From fdb61345c0c5d17c558ac8fa1bc7d004c8419246 Mon Sep 17 00:00:00 2001 From: reshke Date: Tue, 17 Jun 2025 18:30:50 +0000 Subject: [PATCH 01/45] Add debian for PostgreSQL 18 --- debian/changelog | 68 ++++ debian/clean | 3 + debian/control | 354 ++++++++++++++++++ debian/copyright | 270 +++++++++++++ debian/gitlab-ci.yml | 8 + debian/libecpg-compat3.install | 1 + debian/libecpg-compat3.symbols | 44 +++ debian/libecpg-dev.install | 18 + debian/libecpg6.install | 2 + debian/libecpg6.symbols | 31 ++ debian/libpgtypes3.install | 1 + debian/libpgtypes3.symbols | 48 +++ debian/libpq-dev.dirs | 1 + debian/libpq-dev.install | 14 + debian/libpq-oauth.install | 1 + debian/libpq5.install | 2 + debian/libpq5.symbols | 213 +++++++++++ debian/patches/50-per-version-dirs.patch | 29 ++ .../patches/51-default-sockets-in-var.patch | 20 + debian/patches/52-tutorial-README.patch | 16 + .../53-pg_service.conf_directory_doc.patch | 19 + ...bian-alternatives-for-external-tools.patch | 28 ++ debian/patches/70-history | 13 + debian/patches/autoconf2.69 | 7 + debian/patches/filter-debug-prefix-map | 34 ++ debian/patches/focal-arm64-outline-atomics | 23 ++ debian/patches/hurd-iovec | 16 + debian/patches/jit-s390x | 96 +++++ debian/patches/libpgport-pkglibdir | 82 ++++ debian/patches/move-pages32 | 49 +++ .../pgstat-report-conflicts-immediately.patch | 37 ++ debian/patches/series | 14 + debian/po/POTFILES.in | 1 + debian/po/ca.po | 38 ++ debian/po/de.po | 37 ++ debian/po/es.po | 58 +++ debian/po/fr.po | 39 ++ debian/po/it.po | 37 ++ debian/po/nl.po | 40 ++ debian/po/pt.po | 39 ++ debian/po/pt_BR.po | 37 ++ debian/po/ro.po | 50 +++ debian/po/ru.po | 39 ++ debian/po/sv.po | 36 ++ debian/po/tr.po | 41 ++ debian/postgresql-18-jit.install | 3 + debian/postgresql-18.install | 60 +++ debian/postgresql-18.lintian-overrides | 20 + debian/postgresql-18.postinst | 13 + debian/postgresql-18.postrm | 80 ++++ debian/postgresql-18.preinst | 18 + debian/postgresql-18.prerm | 16 + debian/postgresql-18.templates | 7 + debian/postgresql-client-18.install | 48 +++ debian/postgresql-client-18.lintian-overrides | 3 + debian/postgresql-client-18.postinst | 13 + debian/postgresql-client-18.prerm | 12 + debian/postgresql-doc-18.doc-base | 18 + debian/postgresql-doc-18.install | 2 + debian/postgresql-doc-18.postinst | 30 ++ debian/postgresql-doc-18.prerm | 18 + debian/postgresql-plperl-18.install | 3 + debian/postgresql-plperl-18.lintian-overrides | 1 + debian/postgresql-plpython3-18.install | 3 + .../postgresql-plpython3-18.lintian-overrides | 1 + debian/postgresql-pltcl-18.install | 3 + debian/postgresql-pltcl-18.lintian-overrides | 1 + debian/postgresql-server-dev-18.install | 5 + ...postgresql-server-dev-18.lintian-overrides | 1 + debian/rules | 5 + debian/source/format | 1 + debian/source/lintian-overrides | 4 + debian/tests/Makefile.regress | 5 + debian/tests/control | 21 ++ debian/tests/installcheck | 41 ++ debian/tests/run-testsuite | 5 + debian/watch | 4 + 77 files changed, 2519 insertions(+) create mode 100644 debian/changelog create mode 100644 debian/clean create mode 100644 debian/control create mode 100644 debian/copyright create mode 100644 debian/gitlab-ci.yml create mode 100644 debian/libecpg-compat3.install create mode 100644 debian/libecpg-compat3.symbols create mode 100644 debian/libecpg-dev.install create mode 100644 debian/libecpg6.install create mode 100644 debian/libecpg6.symbols create mode 100644 debian/libpgtypes3.install create mode 100644 debian/libpgtypes3.symbols create mode 100644 debian/libpq-dev.dirs create mode 100755 debian/libpq-dev.install create mode 100644 debian/libpq-oauth.install create mode 100644 debian/libpq5.install create mode 100644 debian/libpq5.symbols create mode 100644 debian/patches/50-per-version-dirs.patch create mode 100644 debian/patches/51-default-sockets-in-var.patch create mode 100644 debian/patches/52-tutorial-README.patch create mode 100644 debian/patches/53-pg_service.conf_directory_doc.patch create mode 100644 debian/patches/54-debian-alternatives-for-external-tools.patch create mode 100644 debian/patches/70-history create mode 100644 debian/patches/autoconf2.69 create mode 100644 debian/patches/filter-debug-prefix-map create mode 100644 debian/patches/focal-arm64-outline-atomics create mode 100644 debian/patches/hurd-iovec create mode 100644 debian/patches/jit-s390x create mode 100644 debian/patches/libpgport-pkglibdir create mode 100644 debian/patches/move-pages32 create mode 100644 debian/patches/pgstat-report-conflicts-immediately.patch create mode 100644 debian/patches/series create mode 100644 debian/po/POTFILES.in create mode 100644 debian/po/ca.po create mode 100644 debian/po/de.po create mode 100644 debian/po/es.po create mode 100644 debian/po/fr.po create mode 100644 debian/po/it.po create mode 100644 debian/po/nl.po create mode 100644 debian/po/pt.po create mode 100644 debian/po/pt_BR.po create mode 100644 debian/po/ro.po create mode 100644 debian/po/ru.po create mode 100644 debian/po/sv.po create mode 100644 debian/po/tr.po create mode 100644 debian/postgresql-18-jit.install create mode 100755 debian/postgresql-18.install create mode 100644 debian/postgresql-18.lintian-overrides create mode 100644 debian/postgresql-18.postinst create mode 100644 debian/postgresql-18.postrm create mode 100644 debian/postgresql-18.preinst create mode 100644 debian/postgresql-18.prerm create mode 100644 debian/postgresql-18.templates create mode 100644 debian/postgresql-client-18.install create mode 100644 debian/postgresql-client-18.lintian-overrides create mode 100644 debian/postgresql-client-18.postinst create mode 100644 debian/postgresql-client-18.prerm create mode 100644 debian/postgresql-doc-18.doc-base create mode 100644 debian/postgresql-doc-18.install create mode 100644 debian/postgresql-doc-18.postinst create mode 100644 debian/postgresql-doc-18.prerm create mode 100644 debian/postgresql-plperl-18.install create mode 120000 debian/postgresql-plperl-18.lintian-overrides create mode 100644 debian/postgresql-plpython3-18.install create mode 120000 debian/postgresql-plpython3-18.lintian-overrides create mode 100644 debian/postgresql-pltcl-18.install create mode 120000 debian/postgresql-pltcl-18.lintian-overrides create mode 100644 debian/postgresql-server-dev-18.install create mode 120000 debian/postgresql-server-dev-18.lintian-overrides create mode 100755 debian/rules create mode 100644 debian/source/format create mode 100644 debian/source/lintian-overrides create mode 100644 debian/tests/Makefile.regress create mode 100644 debian/tests/control create mode 100755 debian/tests/installcheck create mode 100755 debian/tests/run-testsuite create mode 100644 debian/watch diff --git a/debian/changelog b/debian/changelog new file mode 100644 index 00000000000..d91159ebb51 --- /dev/null +++ b/debian/changelog @@ -0,0 +1,68 @@ +postgresql-18 (18~beta1+20250624-1.pgdg+1) sid-pgdg; urgency=medium + + * Rebuild for sid-pgdg. + * Changes applied by generate-pgdg-source: + + Moving binary packages to component 18. + + Enabling cassert. + + -- PostgreSQL on Debian and Ubuntu Mon, 23 Jun 2025 14:37:14 +0200 + +postgresql-18 (18~beta1+20250624-1) experimental; urgency=medium + + * New upstream snapshot. + * Restrict libpq-oauth and B-D: libnuma-dev to [linux-any]. + * Work around a Linux 32-bit bug in move_pages on 64-bit kernels. + * Add Turkish debconf translation by Atila KOÇ, thanks! (Closes: #1107984) + * Add Catalan debconf translation by Carles Pina i Estany, thanks! + + -- Christoph Berg Mon, 23 Jun 2025 14:37:14 +0200 + +postgresql-18 (18~beta1+20250612-1) experimental; urgency=medium + + * New upstream snapshot. + * Add B-D on libnuma-dev. + + -- Christoph Berg Fri, 06 Jun 2025 14:29:17 +0200 + +postgresql-18 (18~beta1-1) experimental; urgency=medium + + * First beta version. + + -- Christoph Berg Tue, 06 May 2025 20:28:58 +0200 + +postgresql-18 (18~~devel.20250502-1) experimental; urgency=medium + + * Split libpq-oauth into a separate package so libpq5 does not have to + depend on libcurl. + + -- Christoph Berg Fri, 02 May 2025 10:39:45 +0200 + +postgresql-18 (18~~devel.20250421-1) experimental; urgency=medium + + * New upstream snapshot. + + -- Christoph Berg Mon, 21 Apr 2025 21:07:47 +0200 + +postgresql-18 (18~~devel.20250405-1) experimental; urgency=medium + + * New upstream snapshot. + * B-D on liburing-dev. + + -- Christoph Berg Wed, 02 Apr 2025 15:15:38 +0200 + +postgresql-18 (18~~devel.20250331-1) experimental; urgency=medium + + * New upstream snapshot. + * Drop extension_destdir patch, implemented upstream as + extension_control_path. + * Disable JIT on loong64 and riscv64 again, still segfaulting. + + -- Christoph Berg Wed, 19 Mar 2025 15:47:26 +0100 + +postgresql-18 (18~~devel.20250318+g4078da6c478-1) experimental; urgency=medium + + * New major upstream version 18; packaging based on postgresql-17. + * Move JIT to new postgresql-18-jit package. (Closes: #927182) + * Enable JIT only on 64-bit architectures. + + -- Christoph Berg Tue, 18 Mar 2025 16:43:43 +0100 diff --git a/debian/clean b/debian/clean new file mode 100644 index 00000000000..a1e3d37a58a --- /dev/null +++ b/debian/clean @@ -0,0 +1,3 @@ +# regression tests debris +src/test/regress/regress.o +src/test/regress/regress.so diff --git a/debian/control b/debian/control new file mode 100644 index 00000000000..0e0fb1f87d1 --- /dev/null +++ b/debian/control @@ -0,0 +1,354 @@ +Source: postgresql-18 +Section: 18/database +Priority: optional +Maintainer: Debian PostgreSQL Maintainers +Uploaders: + Martin Pitt , + Peter Eisentraut , + Christoph Berg , +Standards-Version: 4.7.0 +Rules-Requires-Root: no +Build-Depends: + autoconf, + bison, + clang [amd64 arm64 mips64el ppc64 ppc64el s390x] , + debhelper-compat (= 13), + dh-exec (>= 0.13~), + docbook-xml, + docbook-xsl (>= 1.77), + dpkg-dev (>= 1.16.1~), + flex, + gdb , + gettext, + libcurl4-openssl-dev [linux-any], + libicu-dev, + libio-pty-perl , + libipc-run-perl , + libkrb5-dev, + libldap2-dev, + liblz4-dev, + libnuma-dev [linux-any], + libpam0g-dev | libpam-dev, + libperl-dev, + libreadline-dev, + libselinux1-dev [linux-any], + libssl-dev, + libsystemd-dev [linux-any], + liburing-dev [linux-any], + libxml2-dev, + libxml2-utils, + libxslt1-dev, + libzstd-dev (>= 1.4.0) , + llvm-dev [amd64 arm64 mips64el ppc64 ppc64el s390x] , + lz4 | liblz4-tool, + mawk, + perl (>= 5.8), + pkgconf, + postgresql-common (>= 279~), + python3-dev, + systemtap-sdt-dev [linux-any], + tcl-dev, + tzdata , + tzdata-legacy | tzdata (<< 2023c-8) , + uuid-dev, + xsltproc, + zlib1g-dev | libz-dev, + zstd (>= 1.4.0) , +Homepage: http://www.postgresql.org/ +Vcs-Browser: https://salsa.debian.org/postgresql/postgresql +Vcs-Git: https://salsa.debian.org/postgresql/postgresql.git -b 18 + +Package: libpq-dev +Build-Profiles: +Architecture: any +Section: 18/libdevel +Depends: + libpq5 (= ${binary:Version}), + libssl-dev, + ${misc:Depends}, + ${shlibs:Depends}, +Suggests: + postgresql-doc-18, +Description: header files for libpq5 (PostgreSQL library) + Header files and static library for compiling C programs to link + with the libpq library in order to communicate with a PostgreSQL + database backend. + . + PostgreSQL is an object-relational SQL database management system. + +Package: libpq5 +Build-Profiles: +Architecture: any +Section: 18/libs +Depends: + ${misc:Depends}, + ${shlibs:Depends}, +Pre-Depends: + ${misc:Pre-Depends}, +Recommends: + ca-certificates, +Suggests: + libpq-oauth, +Multi-Arch: same +Description: PostgreSQL C client library + libpq is a C library that enables user programs to communicate with + the PostgreSQL database server. The server can be on another machine + and accessed through TCP/IP. This version of libpq is compatible + with servers from PostgreSQL 8.2 or later. + . + This package contains the run-time library, needed by packages using + libpq. SSL certificate validation (the sslrootcert=system connection + option) requires the ca-certificates package. + . + PostgreSQL is an object-relational SQL database management system. + +Package: libpq-oauth +Build-Profiles: +Architecture: linux-any +Section: 18/libs +Depends: + libpq5 (= ${binary:Version}), + ${misc:Depends}, + ${shlibs:Depends}, +Pre-Depends: + ${misc:Pre-Depends}, +Provides: libpq-oauth-18 +Multi-Arch: same +Description: PostgreSQL C client library - OAuth flow + libpq-oauth is an optional module for libpq5 implementing the Device + Authorization flow for OAuth clients (RFC 8628). It is maintained as + its own shared library in order to isolate its dependency on libcurl. + (End users who don't want the Curl dependency can simply choose not + to install this module.) + . + If a connection string allows the use of OAuth, and the server asks + for it, and a libpq client has not installed its own custom OAuth + flow, libpq will attempt to delay-load this module using dlopen() and + the following ABI. Failure to load results in a failed connection. + . + PostgreSQL is an object-relational SQL database management system. + +Package: libecpg6 +Build-Profiles: +Architecture: any +Section: 18/libs +Depends: + ${misc:Depends}, + ${shlibs:Depends}, +Pre-Depends: + ${misc:Pre-Depends}, +Multi-Arch: same +Description: run-time library for ECPG programs + The libecpg shared library is used by programs built with ECPG + (Embedded PostgreSQL for C). + . + PostgreSQL is an object-relational SQL database management system. + +Package: libecpg-dev +Build-Profiles: +Architecture: any +Section: 18/libdevel +Depends: + libecpg-compat3 (= ${binary:Version}), + libecpg6 (= ${binary:Version}), + libpgtypes3 (= ${binary:Version}), + libpq-dev, + ${misc:Depends}, + ${shlibs:Depends}, +Description: development files for ECPG (Embedded PostgreSQL for C) + This package contains the necessary files to build ECPG (Embedded + PostgreSQL for C) programs. It includes the development libraries + and the preprocessor program ecpg. + . + PostgreSQL is an object-relational SQL database management system. + . + Install this package if you want to write C programs with SQL statements + embedded in them (rather than run by an external process). + +Package: libecpg-compat3 +Build-Profiles: +Architecture: any +Section: 18/libs +Depends: + ${misc:Depends}, + ${shlibs:Depends}, +Pre-Depends: + ${misc:Pre-Depends}, +Multi-Arch: same +Description: older version of run-time library for ECPG programs + The libecpg_compat shared library is used by programs built with ecpg. + (Embedded PostgreSQL for C). + . + PostgreSQL is an object-relational SQL database management system. + +Package: libpgtypes3 +Build-Profiles: +Architecture: any +Section: 18/libs +Depends: + ${misc:Depends}, + ${shlibs:Depends}, +Pre-Depends: + ${misc:Pre-Depends}, +Multi-Arch: same +Description: shared library libpgtypes for PostgreSQL 18 + The libpgtypes shared library is used by programs built with ecpg. + (Embedded PostgreSQL for C). + . + PostgreSQL is an object-relational SQL database management system. + +Package: postgresql-18 +Architecture: any +Depends: + locales | locales-all, + postgresql-client-18 (= ${binary:Version}), + postgresql-common (>= 275~), + ssl-cert, + tzdata, + ${misc:Depends}, + ${shlibs:Depends}, +Provides: + postgresql-contrib-18, +Recommends: + postgresql-18-jit, + sysstat, +Breaks: + dbconfig-common (<< 2.0.22~), +Description: The World's Most Advanced Open Source Relational Database + PostgreSQL, also known as Postgres, is a free and open-source relational + database management system (RDBMS) emphasizing extensibility and SQL + compliance. It features transactions with Atomicity, Consistency, Isolation, + Durability (ACID) properties, automatically updatable views, materialized + views, triggers, foreign keys, and stored procedures. It is designed to handle + a range of workloads, from single machines to data warehouses or Web services + with many concurrent users. + . + This package provides the database server for PostgreSQL 18.${cassert} +XB-Postgresql-Catversion: ${postgresql:Catversion} + +Package: postgresql-client-18 +Architecture: any +Multi-Arch: foreign +Depends: + libpq5 (>= ${source:Upstream-Version}), + postgresql-client-common (>= 182~), + sensible-utils, + ${misc:Depends}, + ${shlibs:Depends}, +Suggests: + postgresql-18, + postgresql-doc-18, +Provides: + postgresql-client, +Description: front-end programs for PostgreSQL 18 + This package contains client and administrative programs for + PostgreSQL: these are the interactive terminal client psql and + programs for creating and removing users and databases. + . + This is the client package for PostgreSQL 18. If you install + PostgreSQL 18 on a standalone machine, you need the server package + postgresql-18, too. On a network, you can install this package on + many client machines, while the server package may be installed on + only one machine. + . + PostgreSQL is an object-relational SQL database management system. + +Package: postgresql-server-dev-18 +Architecture: any +Section: 18/libdevel +Depends: + clang-${llvm:Version} [amd64 arm64 mips64el ppc64 ppc64el s390x] , + libpq-dev (>= 18~~), + llvm-${llvm:Version}-dev [amd64 arm64 mips64el ppc64 ppc64el s390x] , + postgresql-client-18 (= ${binary:Version}), + postgresql-common (>= 142~), + ${misc:Depends}, + ${shlibs:Depends}, +Description: development files for PostgreSQL 18 server-side programming + Header files for compiling SPI code to link into PostgreSQL's backend; for + example, for C functions to be called from SQL. + . + This package also contains the Makefiles necessary for building add-on + modules of PostgreSQL, which would otherwise have to be built in the + PostgreSQL source-code tree. + . + PostgreSQL is an object-relational SQL database management system. + +Package: postgresql-doc-18 +Architecture: all +Build-Profiles: +Multi-Arch: foreign +Section: 18/doc +Depends: + ${misc:Depends}, +Description: documentation for the PostgreSQL database management system + This package contains all README files, user manual, and examples for + PostgreSQL 18. The manual is in HTML format. + . + PostgreSQL is an object-relational SQL database management system. + +Package: postgresql-plperl-18 +Architecture: any +Depends: + perl, + postgresql-18 (= ${binary:Version}), + ${misc:Depends}, + ${shlibs:Depends}, +Provides: + postgresql-plperl, +Description: PL/Perl procedural language for PostgreSQL 18 + PL/Perl enables an SQL developer to write procedural language functions + for PostgreSQL 18 in Perl. You need this package if you have any + PostgreSQL 18 functions that use the languages plperl or plperlu. + . + PostgreSQL is an object-relational SQL database management system. + +Package: postgresql-plpython3-18 +Architecture: any +Depends: + postgresql-18 (= ${binary:Version}), + ${misc:Depends}, + ${shlibs:Depends}, +Provides: + postgresql-plpython3, +Description: PL/Python 3 procedural language for PostgreSQL 18 + PL/Python 3 enables an SQL developer to write procedural language functions + for PostgreSQL 18 in Python 3. You need this package if you have any + PostgreSQL 18 functions that use the languages plpython3 or plpython3u. + . + PostgreSQL is an object-relational SQL database management system. + +Package: postgresql-pltcl-18 +Architecture: any +Depends: + postgresql-18 (= ${binary:Version}), + ${misc:Depends}, + ${shlibs:Depends}, +Provides: + postgresql-pltcl, +Description: PL/Tcl procedural language for PostgreSQL 18 + PL/Tcl enables an SQL developer to write procedural language functions + for PostgreSQL 18 in Tcl. You need this package if you have any + PostgreSQL 18 functions that use the languages pltcl or pltclu. + . + PostgreSQL is an object-relational SQL database management system. + +Package: postgresql-18-jit +Architecture: amd64 arm64 mips64el ppc64 ppc64el s390x +Build-Profiles: +Depends: + postgresql-18 (= ${binary:Version}), + ${misc:Depends}, + ${shlibs:Depends}, +Provides: + postgresql-18-jit-llvm (= ${llvm:Version}), +Description: LLVM JIT support for PostgreSQL 18 + PostgreSQL, also known as Postgres, is a free and open-source relational + database management system (RDBMS) emphasizing extensibility and SQL + compliance. It features transactions with Atomicity, Consistency, Isolation, + Durability (ACID) properties, automatically updatable views, materialized + views, triggers, foreign keys, and stored procedures. It is designed to handle + a range of workloads, from single machines to data warehouses or Web services + with many concurrent users. + . + This package provides LLVM-based query JITing for PostgreSQL 18. diff --git a/debian/copyright b/debian/copyright new file mode 100644 index 00000000000..df27d1ff61e --- /dev/null +++ b/debian/copyright @@ -0,0 +1,270 @@ +Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Upstream-Name: PostgreSQL +Source: https://www.postgresql.org/ftp/source/ + +### PostgreSQL copyrights ### + +Files: * +Copyright: Portions Copyright (c) 1996-2019, PostgreSQL Global Development Group + Portions Copyright (c) 1994, The Regents of the University of California +License: PostgreSQL + +Files: src/backend/regex/* +Copyright: + Copyright (c) 2013-2022, PostgreSQL Global Development Group + Copyright (c) 1998, 1999 Henry Spencer. All rights reserved. + Copyright (c) 1998 Sun Microsystems, Inc. + Copyright (c) 1998, 1999 by Scriptics Corporation. +License: PostgreSQL and Custom-regex and Tcl + +Files: src/bin/pg_dump/* +Copyright: Portions Copyright (c) 2000, Philip Warner +License: Custom-pg_dump + Rights are granted to use this software in any way so long as this notice is + not removed. The author is not responsible for loss or damages that may + result from its use. + +Files: src/backend/snowball/libstemmer/* src/include/snowball/libstemmer/* +Copyright: Copyright (c) 2001, Dr Martin Porter, + Copyright (c) 2002, Richard Boulton. +License: BSD-3-Clause +Comment: + This module uses the word stemming code developed by the Snowball project, + http://snowballstem.org which is released by them under a BSD-style license. + Postgres' files under src/backend/snowball/libstemmer/ and + src/include/snowball/libstemmer/ are taken directly from the Snowball files, + with only some minor adjustments of file inclusions. + See src/backend/snowball/README for details. + +Files: src/backend/utils/mb/Unicode/*.txt +Copyright: Copyright (C) 2001 earthian@tama.or.jp, All Rights Reserved. + Copyright (C) 2001 I'O, All Rights Reserved. + Copyright (C) 2006 Project X0213, All Rights Reserved. +License: Custom-Unicode + You can use, modify, distribute this table freely. + +Files: + src/common/sha2.c + src/include/common/sha2.h +Copyright: Copyright (c) 2000-2001, Aaron D. Gifford +License: BSD-3-Clause + +### contrib copyrights ### + +Files: contrib/fuzzystrmatch/dmetaphone.c +Copyright: Copyright 2000, Maurice Aubrey + Copyright 2003, North Carolina State Highway Patrol +License: double-metaphone + This module is free software; you may redistribute it and/or + modify it under the same terms as Perl itself. + . + All rights reserved. + . + Permission to use, copy, modify, and distribute this software and its + documentation for any purpose, without fee, and without a written agreement + is hereby granted, provided that the above copyright notice and this + paragraph and the following two paragraphs appear in all copies. + . + IN NO EVENT SHALL THE NORTH CAROLINA STATE HIGHWAY PATROL BE LIABLE TO ANY + PARTY FOR DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, + INCLUDING LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS + DOCUMENTATION, EVEN IF THE NORTH CAROLINA STATE HIGHWAY PATROL HAS BEEN + ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + . + THE NORTH CAROLINA STATE HIGHWAY PATROL SPECIFICALLY DISCLAIMS ANY + WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED + HEREUNDER IS ON AN "AS IS" BASIS, AND THE NORTH CAROLINA STATE HIGHWAY PATROL + HAS NO OBLIGATIONS TO PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR + MODIFICATIONS. + . + The license of Perl is: + . + This program is free software; you can redistribute it and/or modify + it under the terms of either: + . + a) the GNU General Public License as published by the Free Software + Foundation; either version 1, or (at your option) any later + version, or + . + b) the "Artistic License" which comes with Perl. + . + On Debian GNU/Linux systems, the complete text of the GNU General + Public License version 1 can be found in + `/usr/share/common-licenses/GPL-1' and the Artistic Licence in + `/usr/share/common-licenses/Artistic'. + +Files: + contrib/pageinspect/btreefuncs.c + contrib/pgrowlocks/* + contrib/pgstattuple/* +Copyright: Copyright (c) 2006 Satoshi Nagayasu + Copyright (c) 2001, 2002, 2005-2006 Tatsuo Ishii +License: nagaysau-ishii + Permission to use, copy, modify, and distribute this software and + its documentation for any purpose, without fee, and without a + written agreement is hereby granted, provided that the above + copyright notice and this paragraph and the following two + paragraphs appear in all copies. + . + IN NO EVENT SHALL THE AUTHOR BE LIABLE TO ANY PARTY FOR DIRECT, + INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS + DOCUMENTATION, EVEN IF THE UNIVERSITY OF CALIFORNIA HAS BEEN ADVISED + OF THE POSSIBILITY OF SUCH DAMAGE. + . + THE AUTHOR SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON AN "AS + IS" BASIS, AND THE AUTHOR HAS NO OBLIGATIONS TO PROVIDE MAINTENANCE, + SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + +Files: + contrib/pgcrypto/crypt-des.c +Copyright: Copyright (c) 1994 David Burren + Copyright (C) 1995, 1996, 1997, and 1998 WIDE Project. +License: BSD-3-clause + +Files: + contrib/pgcrypto/mbuf.* + contrib/pgcrypto/openssl.c + contrib/pgcrypto/pgcrypto.* + contrib/pgcrypto/pgp* + contrib/pgcrypto/px* +Copyright: Copyright (c) 2001, 2005 Marko Kreen +License: BSD-2-clause + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + . + THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + SUCH DAMAGE. + +### licenses ### + +License: PostgreSQL + Permission to use, copy, modify, and distribute this software and its + documentation for any purpose, without fee, and without a written agreement + is hereby granted, provided that the above copyright notice and this + paragraph and the following two paragraphs appear in all copies. + . + IN NO EVENT SHALL THE UNIVERSITY OF CALIFORNIA BE LIABLE TO ANY PARTY FOR + DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS + DOCUMENTATION, EVEN IF THE UNIVERSITY OF CALIFORNIA HAS BEEN ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + . + THE UNIVERSITY OF CALIFORNIA SPECIFICALLY DISCLAIMS ANY WARRANTIES, + INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS + ON AN "AS IS" BASIS, AND THE UNIVERSITY OF CALIFORNIA HAS NO OBLIGATIONS TO + PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + +License: BSD-3-Clause + All rights reserved. + . + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + . + 1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + . + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + . + 3. Neither the name of the copyright holder nor the names of its contributors + may be used to endorse or promote products derived from this software without + specific prior written permission. + . + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +License: Custom-regex + Development of this software was funded, in part, by Cray Research Inc., + UUNET Communications Services Inc., Sun Microsystems Inc., and Scriptics + Corporation, none of whom are responsible for the results. The author + thanks all of them. + . + Redistribution and use in source and binary forms -- with or without + modification -- are permitted for any purpose, provided that + redistributions in source form retain this entire copyright notice and + indicate the origin and nature of any modifications. + . + I'd appreciate being given credit for this package in the documentation + of software which uses it, but that is not a requirement. + . + THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, + INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL + HENRY SPENCER BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +License: Tcl + This software is copyrighted by the Regents of the University of + California, Sun Microsystems, Inc., Scriptics Corporation, ActiveState + Corporation and other parties. The following terms apply to all files + associated with the software unless explicitly disclaimed in + individual files. + . + The authors hereby grant permission to use, copy, modify, distribute, + and license this software and its documentation for any purpose, provided + that existing copyright notices are retained in all copies and that this + notice is included verbatim in any distributions. No written agreement, + license, or royalty fee is required for any of the authorized uses. + Modifications to this software may be copyrighted by their authors + and need not follow the licensing terms described here, provided that + the new terms are clearly indicated on the first page of each file where + they apply. + . + IN NO EVENT SHALL THE AUTHORS OR DISTRIBUTORS BE LIABLE TO ANY PARTY + FOR DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES + ARISING OUT OF THE USE OF THIS SOFTWARE, ITS DOCUMENTATION, OR ANY + DERIVATIVES THEREOF, EVEN IF THE AUTHORS HAVE BEEN ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + . + THE AUTHORS AND DISTRIBUTORS SPECIFICALLY DISCLAIM ANY WARRANTIES, + INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE, AND NON-INFRINGEMENT. THIS SOFTWARE + IS PROVIDED ON AN "AS IS" BASIS, AND THE AUTHORS AND DISTRIBUTORS HAVE + NO OBLIGATION TO PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR + MODIFICATIONS. + . + GOVERNMENT USE: If you are acquiring this software on behalf of the + U.S. government, the Government shall have only "Restricted Rights" + in the software and related documentation as defined in the Federal + Acquisition Regulations (FARs) in Clause 52.227.19 (c) (2). If you + are acquiring the software on behalf of the Department of Defense, the + software shall be classified as "Commercial Computer Software" and the + Government shall have only "Restricted Rights" as defined in Clause + 252.227-7013 (c) (1) of DFARs. Notwithstanding the foregoing, the + authors grant the U.S. Government and others acting in its behalf + permission to use and distribute the software in accordance with the + terms specified in this license. diff --git a/debian/gitlab-ci.yml b/debian/gitlab-ci.yml new file mode 100644 index 00000000000..84a83b0d7a0 --- /dev/null +++ b/debian/gitlab-ci.yml @@ -0,0 +1,8 @@ +include: https://salsa.debian.org/postgresql/postgresql-common/raw/master/gitlab/gitlab-ci.yml + +variables: + # disable piuparts because it doesn't like testing (lib) packages in an environment where newer versions from unstable exist + #SALSA_CI_DISABLE_PIUPARTS: 'yes' + # does not understand our use of dpkg-divert + SALSA_CI_DISABLE_MISSING_BREAKS: 1 + RELEASE: 'experimental' diff --git a/debian/libecpg-compat3.install b/debian/libecpg-compat3.install new file mode 100644 index 00000000000..c69d50d8467 --- /dev/null +++ b/debian/libecpg-compat3.install @@ -0,0 +1 @@ +usr/lib/*/libecpg_compat.so.3* diff --git a/debian/libecpg-compat3.symbols b/debian/libecpg-compat3.symbols new file mode 100644 index 00000000000..3bd41e03e15 --- /dev/null +++ b/debian/libecpg-compat3.symbols @@ -0,0 +1,44 @@ +libecpg_compat.so.3 libecpg-compat3 #MINVER# +* Build-Depends-Package: libecpg-dev + ECPG_informix_get_var@Base 0 + ECPG_informix_reset_sqlca@Base 9.0~ + ECPG_informix_set_var@Base 0 + byleng@Base 0 + decadd@Base 0 + deccmp@Base 0 + deccopy@Base 0 + deccvasc@Base 0 + deccvdbl@Base 0 + deccvint@Base 0 + deccvlong@Base 0 + decdiv@Base 0 + decmul@Base 0 + decsub@Base 0 + dectoasc@Base 0 + dectodbl@Base 0 + dectoint@Base 0 + dectolong@Base 0 + dtcurrent@Base 0 + dtcvasc@Base 0 + dtcvfmtasc@Base 0 + dtsub@Base 0 + dttoasc@Base 0 + dttofmtasc@Base 0 + intoasc@Base 0 + ldchar@Base 0 + rdatestr@Base 0 + rdayofweek@Base 0 + rdefmtdate@Base 0 + rfmtdate@Base 0 + rfmtlong@Base 0 + rgetmsg@Base 0 + risnull@Base 0 + rjulmdy@Base 0 + rmdyjul@Base 0 + rsetnull@Base 0 + rstrdate@Base 0 + rtoday@Base 0 + rtypalign@Base 0 + rtypmsize@Base 0 + rtypwidth@Base 0 + rupshift@Base 0 diff --git a/debian/libecpg-dev.install b/debian/libecpg-dev.install new file mode 100644 index 00000000000..1935a0d885a --- /dev/null +++ b/debian/libecpg-dev.install @@ -0,0 +1,18 @@ +usr/include/postgresql/ecpg*.h +usr/include/postgresql/informix/* +usr/include/postgresql/pgtypes*.h +usr/include/postgresql/sql3types.h +usr/include/postgresql/sqlca.h +usr/include/postgresql/sqlda*.h +usr/lib/*/libecpg.a +usr/lib/*/libecpg.so +usr/lib/*/libecpg_compat.a +usr/lib/*/libecpg_compat.so +usr/lib/*/libpgtypes.a +usr/lib/*/libpgtypes.so +usr/lib/*/pkgconfig/libecpg.pc +usr/lib/*/pkgconfig/libecpg_compat.pc +usr/lib/*/pkgconfig/libpgtypes.pc +usr/lib/postgresql/*/bin/ecpg usr/bin +usr/share/locale/*/*/ecpg-*.mo +usr/share/postgresql/*/man/man1/ecpg.1* /usr/share/man/man1 diff --git a/debian/libecpg6.install b/debian/libecpg6.install new file mode 100644 index 00000000000..8ec1cee0f03 --- /dev/null +++ b/debian/libecpg6.install @@ -0,0 +1,2 @@ +usr/lib/*/libecpg.so.6* +usr/share/locale/*/*/ecpglib*.mo diff --git a/debian/libecpg6.symbols b/debian/libecpg6.symbols new file mode 100644 index 00000000000..d20881adac9 --- /dev/null +++ b/debian/libecpg6.symbols @@ -0,0 +1,31 @@ +libecpg.so.6 libecpg6 #MINVER# +* Build-Depends-Package: libecpg-dev + ECPGallocate_desc@Base 0 + ECPGconnect@Base 0 + ECPGdeallocate@Base 0 + ECPGdeallocate_all@Base 0 + ECPGdeallocate_desc@Base 0 + ECPGdebug@Base 0 + ECPGdescribe@Base 0 + ECPGdisconnect@Base 0 + ECPGdo@Base 0 + ECPGdo_descriptor@Base 0 + ECPGfree_auto_mem@Base 0 + ECPGget_PGconn@Base 8.3.1-2~ + ECPGget_desc@Base 0 + ECPGget_desc_header@Base 0 + ECPGget_sqlca@Base 0 + ECPGget_var@Base 9.0~ + ECPGis_noind_null@Base 0 + ECPGprepare@Base 0 + ECPGprepared_statement@Base 0 + ECPGset_desc@Base 0 + ECPGset_desc_header@Base 0 + ECPGset_noind_null@Base 0 + ECPGset_var@Base 9.0~ + ECPGsetcommit@Base 0 + ECPGsetconn@Base 0 + ECPGstatus@Base 0 + ECPGtrans@Base 0 + ECPGtransactionStatus@Base 9.0~ + sqlprint@Base 0 diff --git a/debian/libpgtypes3.install b/debian/libpgtypes3.install new file mode 100644 index 00000000000..decf90ae6bc --- /dev/null +++ b/debian/libpgtypes3.install @@ -0,0 +1 @@ +usr/lib/*/libpgtypes.so.3* diff --git a/debian/libpgtypes3.symbols b/debian/libpgtypes3.symbols new file mode 100644 index 00000000000..a398be3399a --- /dev/null +++ b/debian/libpgtypes3.symbols @@ -0,0 +1,48 @@ +libpgtypes.so.3 libpgtypes3 #MINVER# +* Build-Depends-Package: libecpg-dev + PGTYPESchar_free@Base 11~beta2 + PGTYPESdate_dayofweek@Base 0 + PGTYPESdate_defmt_asc@Base 0 + PGTYPESdate_fmt_asc@Base 0 + PGTYPESdate_free@Base 0 + PGTYPESdate_from_asc@Base 0 + PGTYPESdate_from_timestamp@Base 0 + PGTYPESdate_julmdy@Base 0 + PGTYPESdate_mdyjul@Base 0 + PGTYPESdate_new@Base 0 + PGTYPESdate_to_asc@Base 0 + PGTYPESdate_today@Base 0 + PGTYPESdecimal_free@Base 0 + PGTYPESdecimal_new@Base 0 + PGTYPESinterval_copy@Base 0 + PGTYPESinterval_free@Base 0 + PGTYPESinterval_from_asc@Base 0 + PGTYPESinterval_new@Base 0 + PGTYPESinterval_to_asc@Base 0 + PGTYPESnumeric_add@Base 0 + PGTYPESnumeric_cmp@Base 0 + PGTYPESnumeric_copy@Base 0 + PGTYPESnumeric_div@Base 0 + PGTYPESnumeric_free@Base 0 + PGTYPESnumeric_from_asc@Base 0 + PGTYPESnumeric_from_decimal@Base 0 + PGTYPESnumeric_from_double@Base 0 + PGTYPESnumeric_from_int@Base 0 + PGTYPESnumeric_from_long@Base 0 + PGTYPESnumeric_mul@Base 0 + PGTYPESnumeric_new@Base 0 + PGTYPESnumeric_sub@Base 0 + PGTYPESnumeric_to_asc@Base 0 + PGTYPESnumeric_to_decimal@Base 0 + PGTYPESnumeric_to_double@Base 0 + PGTYPESnumeric_to_int@Base 0 + PGTYPESnumeric_to_long@Base 0 + PGTYPEStimestamp_add_interval@Base 0 + PGTYPEStimestamp_current@Base 0 + PGTYPEStimestamp_defmt_asc@Base 0 + PGTYPEStimestamp_defmt_scan@Base 0 + PGTYPEStimestamp_fmt_asc@Base 0 + PGTYPEStimestamp_from_asc@Base 0 + PGTYPEStimestamp_sub@Base 0 + PGTYPEStimestamp_sub_interval@Base 0 + PGTYPEStimestamp_to_asc@Base 0 diff --git a/debian/libpq-dev.dirs b/debian/libpq-dev.dirs new file mode 100644 index 00000000000..e7724817552 --- /dev/null +++ b/debian/libpq-dev.dirs @@ -0,0 +1 @@ +usr/bin diff --git a/debian/libpq-dev.install b/debian/libpq-dev.install new file mode 100755 index 00000000000..c2fe1bf2f9c --- /dev/null +++ b/debian/libpq-dev.install @@ -0,0 +1,14 @@ +#!/usr/bin/dh-exec + +usr/include/postgresql/internal/* +usr/include/postgresql/libpq-events.h +usr/include/postgresql/libpq-fe.h +usr/include/postgresql/libpq/libpq-fs.h +usr/include/postgresql/pg_config*.h +usr/include/postgresql/postgres_ext.h +usr/lib/*/libpq.a +usr/lib/*/libpq.so +[linux-any] usr/lib/*/libpq-oauth.a +usr/lib/*/pkgconfig/libpq.pc +# pg_config manpage for both the perl and C versions +usr/share/postgresql/*/man/man1/pg_config.1* /usr/share/man/man1 diff --git a/debian/libpq-oauth.install b/debian/libpq-oauth.install new file mode 100644 index 00000000000..c7fd69bc380 --- /dev/null +++ b/debian/libpq-oauth.install @@ -0,0 +1 @@ +usr/lib/*/libpq-oauth-*.so diff --git a/debian/libpq5.install b/debian/libpq5.install new file mode 100644 index 00000000000..c9811d7d9f1 --- /dev/null +++ b/debian/libpq5.install @@ -0,0 +1,2 @@ +usr/lib/*/libpq.so.5* +usr/share/locale/*/LC_MESSAGES/libpq*.mo diff --git a/debian/libpq5.symbols b/debian/libpq5.symbols new file mode 100644 index 00000000000..488be5f18cb --- /dev/null +++ b/debian/libpq5.symbols @@ -0,0 +1,213 @@ +libpq.so.5 libpq5 #MINVER# +* Build-Depends-Package: libpq-dev + PQbackendPID@Base 0 + PQbinaryTuples@Base 0 + PQcancel@Base 0 + PQcancelBlocking@Base 17~~ + PQcancelCreate@Base 17~~ + PQcancelErrorMessage@Base 17~~ + PQcancelFinish@Base 17~~ + PQcancelPoll@Base 17~~ + PQcancelReset@Base 17~~ + PQcancelSocket@Base 17~~ + PQcancelStart@Base 17~~ + PQcancelStatus@Base 17~~ + PQchangePassword@Base 17~~ + PQclear@Base 0 + PQclientEncoding@Base 0 + PQclosePortal@Base 17~~ + PQclosePrepared@Base 17~~ + PQcmdStatus@Base 0 + PQcmdTuples@Base 0 + PQconndefaults@Base 0 + PQconnectPoll@Base 0 + PQconnectStart@Base 0 + PQconnectStartParams@Base 9.0~ + PQconnectdb@Base 0 + PQconnectdbParams@Base 9.0~ + PQconnectionNeedsPassword@Base 8.3~rc1-1~ + PQconnectionUsedGSSAPI@Base 16~~ + PQconnectionUsedPassword@Base 8.3~ + PQconninfo@Base 9.3~ + PQconninfoFree@Base 0 + PQconninfoParse@Base 8.4~ + PQconsumeInput@Base 0 + PQcopyResult@Base 8.4~ + PQdb@Base 0 + PQdefaultAuthDataHook@Base 18~~ + PQdefaultSSLKeyPassHook_OpenSSL@Base 13~~ + PQdescribePortal@Base 0 + PQdescribePrepared@Base 0 + PQdisplayTuples@Base 0 + PQdsplen@Base 0 + PQencryptPassword@Base 0 + PQencryptPasswordConn@Base 10~~ + PQendcopy@Base 0 + PQenterPipelineMode@Base 14~~ + PQenv2encoding@Base 0 + PQerrorMessage@Base 0 + PQescapeBytea@Base 0 + PQescapeByteaConn@Base 0 + PQescapeIdentifier@Base 9.0~ + PQescapeLiteral@Base 9.0~ + PQescapeString@Base 0 + PQescapeStringConn@Base 0 + PQexec@Base 0 + PQexecParams@Base 0 + PQexecPrepared@Base 0 + PQexitPipelineMode@Base 14~~ + PQfformat@Base 0 + PQfinish@Base 0 + PQfireResultCreateEvents@Base 8.4~ + PQflush@Base 0 + PQfmod@Base 0 + PQfn@Base 0 + PQfname@Base 0 + PQfnumber@Base 0 + PQfreeCancel@Base 0 + PQfreeNotify@Base 0 + PQfreemem@Base 0 + PQfsize@Base 0 + PQftable@Base 0 + PQftablecol@Base 0 + PQftype@Base 0 + PQfullProtocolVersion@Base 18~~ + PQgetAuthDataHook@Base 18~~ + PQgetCancel@Base 0 + PQgetCopyData@Base 0 + PQgetCurrentTimeUSec@Base 17~beta2 + PQgetResult@Base 0 + PQgetSSLKeyPassHook_OpenSSL@Base 13~~ + PQgetgssctx@Base 12~~ + PQgetisnull@Base 0 + PQgetlength@Base 0 + PQgetline@Base 0 + PQgetlineAsync@Base 0 + PQgetssl@Base 0 + PQgetvalue@Base 0 + PQgssEncInUse@Base 12~~ + PQhost@Base 0 + PQhostaddr@Base 12~~ + PQinitOpenSSL@Base 8.4~ + PQinitSSL@Base 0 + PQinstanceData@Base 8.4~ + PQisBusy@Base 0 + PQisnonblocking@Base 0 + PQisthreadsafe@Base 0 + PQlibVersion@Base 9.1~ + PQmakeEmptyPGresult@Base 0 + PQmblen@Base 0 + PQmblenBounded@Base 14~beta2 + PQnfields@Base 0 + PQnotifies@Base 0 + PQnparams@Base 0 + PQntuples@Base 0 + PQoidStatus@Base 0 + PQoidValue@Base 0 + PQoptions@Base 0 + PQparameterStatus@Base 0 + PQparamtype@Base 0 + PQpass@Base 0 + PQping@Base 9.1~ + PQpingParams@Base 9.1~ + PQpipelineStatus@Base 14~~ + PQpipelineSync@Base 14~~ + PQport@Base 0 + PQprepare@Base 0 + PQprint@Base 0 + PQprintTuples@Base 0 + PQprotocolVersion@Base 0 + PQputCopyData@Base 0 + PQputCopyEnd@Base 0 + PQputline@Base 0 + PQputnbytes@Base 0 + PQregisterEventProc@Base 8.4~ + PQregisterThreadLock@Base 0 + PQrequestCancel@Base 0 + PQresStatus@Base 0 + PQreset@Base 0 + PQresetPoll@Base 0 + PQresetStart@Base 0 + PQresultAlloc@Base 8.4~ + PQresultErrorField@Base 0 + PQresultErrorMessage@Base 0 + PQresultInstanceData@Base 8.4~ + PQresultMemorySize@Base 12~~ + PQresultSetInstanceData@Base 8.4~ + PQresultStatus@Base 0 + PQresultVerboseErrorMessage@Base 9.6~~ + PQsendClosePortal@Base 17~~ + PQsendClosePrepared@Base 17~~ + PQsendDescribePortal@Base 0 + PQsendDescribePrepared@Base 0 + PQsendFlushRequest@Base 15~~ + PQsendPipelineSync@Base 17~~ + PQsendPrepare@Base 0 + PQsendQuery@Base 0 + PQsendQueryParams@Base 0 + PQsendQueryPrepared@Base 0 + PQserverVersion@Base 0 + PQservice@Base 18~~ + PQsetAuthDataHook@Base 18~~ + PQsetChunkedRowsMode@Base 17~~ + PQsetClientEncoding@Base 0 + PQsetErrorContextVisibility@Base 9.6~~ + PQsetErrorVerbosity@Base 0 + PQsetInstanceData@Base 8.4~ + PQsetNoticeProcessor@Base 0 + PQsetNoticeReceiver@Base 0 + PQsetResultAttrs@Base 8.4~ + PQsetSSLKeyPassHook_OpenSSL@Base 13~~ + PQsetSingleRowMode@Base 9.2~beta3 + PQsetTraceFlags@Base 14~beta2 + PQsetdbLogin@Base 0 + PQsetnonblocking@Base 0 + PQsetvalue@Base 8.4~ + PQsocket@Base 0 + PQsocketPoll@Base 17~~ + PQsslAttribute@Base 9.5~~ + PQsslAttributeNames@Base 9.5~~ + PQsslInUse@Base 9.5~~ + PQsslStruct@Base 9.5~~ + PQstatus@Base 0 + PQtrace@Base 0 + PQtransactionStatus@Base 0 + PQtty@Base 0 + PQunescapeBytea@Base 0 + PQuntrace@Base 0 + PQuser@Base 0 + appendBinaryPQExpBuffer@Base 0 + appendPQExpBuffer@Base 0 + appendPQExpBufferChar@Base 0 + appendPQExpBufferStr@Base 0 + appendPQExpBufferVA@Base 18~~ + createPQExpBuffer@Base 0 + destroyPQExpBuffer@Base 0 + enlargePQExpBuffer@Base 0 + initPQExpBuffer@Base 0 + lo_close@Base 0 + lo_creat@Base 0 + lo_create@Base 0 + lo_export@Base 0 + lo_import@Base 0 + lo_import_with_oid@Base 8.4~ + lo_lseek64@Base 9.3~ + lo_lseek@Base 0 + lo_open@Base 0 + lo_read@Base 0 + lo_tell64@Base 9.3~ + lo_tell@Base 0 + lo_truncate64@Base 9.3~ + lo_truncate@Base 8.3~ + lo_unlink@Base 0 + lo_write@Base 0 + pg_char_to_encoding@Base 0 + pg_encoding_to_char@Base 0 + pg_utf_mblen@Base 0 + pg_valid_server_encoding@Base 0 + pg_valid_server_encoding_id@Base 8.3~beta1-2~ + pgresStatus@Base 0 + pqsignal@Base 0 + printfPQExpBuffer@Base 0 + resetPQExpBuffer@Base 0 + termPQExpBuffer@Base 0 diff --git a/debian/patches/50-per-version-dirs.patch b/debian/patches/50-per-version-dirs.patch new file mode 100644 index 00000000000..8277cdb127b --- /dev/null +++ b/debian/patches/50-per-version-dirs.patch @@ -0,0 +1,29 @@ +Author: Martin Pitt +Description: Use version specific installation directories so that several major versions can be installed in parallel. +Forwarded: No, Debian specific packaging with postgresql-common + + * Install lib files into /usr/lib/postgresql//lib/ + * Install server related header files into /usr/include/postgresql//server/ + +Bug-Debian: http://bugs.debian.org/462037 + +--- a/src/Makefile.global.in ++++ b/src/Makefile.global.in +@@ -119,7 +119,7 @@ libdir := @libdir@ + pkglibdir = $(libdir) + ifeq "$(findstring pgsql, $(pkglibdir))" "" + ifeq "$(findstring postgres, $(pkglibdir))" "" +-override pkglibdir := $(pkglibdir)/postgresql ++override pkglibdir := /usr/lib/postgresql/@PG_MAJORVERSION@/lib + endif + endif + +@@ -167,7 +167,7 @@ endif # PGXS + + # These derived path variables aren't separately configurable. + +-includedir_server = $(pkgincludedir)/server ++includedir_server = $(pkgincludedir)/@PG_MAJORVERSION@/server + includedir_internal = $(pkgincludedir)/internal + pgxsdir = $(pkglibdir)/pgxs + bitcodedir = $(pkglibdir)/bitcode diff --git a/debian/patches/51-default-sockets-in-var.patch b/debian/patches/51-default-sockets-in-var.patch new file mode 100644 index 00000000000..9da49b833a1 --- /dev/null +++ b/debian/patches/51-default-sockets-in-var.patch @@ -0,0 +1,20 @@ +Author: Martin Pitt +Description: Put server Unix sockets into /var/run/postgresql/ by default +Forwarded: No, Debian specific configuration with postgresql-common + +Using /tmp for sockets allows everyone to spoof a PostgreSQL server. Thus use +/var/run/postgresql/ for "system" clusters which run as 'postgres' (user +clusters will still use /tmp). Since system cluster are by far the common case, +set it as default. + +--- a/src/include/pg_config_manual.h ++++ b/src/include/pg_config_manual.h +@@ -206,7 +206,7 @@ + * support them yet. + */ + #ifndef WIN32 +-#define DEFAULT_PGSOCKET_DIR "/tmp" ++#define DEFAULT_PGSOCKET_DIR "/var/run/postgresql" + #else + #define DEFAULT_PGSOCKET_DIR "" + #endif diff --git a/debian/patches/52-tutorial-README.patch b/debian/patches/52-tutorial-README.patch new file mode 100644 index 00000000000..9eb3263c2b3 --- /dev/null +++ b/debian/patches/52-tutorial-README.patch @@ -0,0 +1,16 @@ +Author: Martin Pitt +Description: Update tutorial README for required build dependencies. +Forwarded: No, Debian specific + +--- a/src/tutorial/README ++++ b/src/tutorial/README +@@ -6,8 +6,7 @@ tutorial + This directory contains SQL tutorial scripts. To look at them, first do a + % make + to compile all the scripts and C files for the user-defined functions +-and types. (make needs to be GNU make --- it may be named something +-different on your system, often 'gmake') ++and types. This requires a postgresql-server-dev-* package to be installed. + + Then, run psql with the -s (single-step) flag: + % psql -s diff --git a/debian/patches/53-pg_service.conf_directory_doc.patch b/debian/patches/53-pg_service.conf_directory_doc.patch new file mode 100644 index 00000000000..584b41c01f8 --- /dev/null +++ b/debian/patches/53-pg_service.conf_directory_doc.patch @@ -0,0 +1,19 @@ +Author: Martin Pitt +Description: Update pg_service.conf example to tell the Debian specific file location. +Forwarded: No, Debian specific + +Index: postgresql-9.2-9.2~beta1/src/interfaces/libpq/pg_service.conf.sample +=================================================================== +--- postgresql-9.2-9.2~beta1.orig/src/interfaces/libpq/pg_service.conf.sample 2011-04-27 23:17:22.000000000 +0200 ++++ postgresql-9.2-9.2~beta1/src/interfaces/libpq/pg_service.conf.sample 2011-05-10 11:25:42.151949794 +0200 +@@ -8,8 +8,8 @@ + # to look up such parameters. A sample configuration for postgres is + # included in this file. Lines beginning with '#' are comments. + # +-# Copy this to your sysconf directory (typically /usr/local/pgsql/etc) and +-# rename it pg_service.conf. ++# Copy this to /etc/postgresql-common/ (or select its location with the ++# PGSYSCONFDIR environment variable) and rename it pg_service.conf. + # + # + #[postgres] diff --git a/debian/patches/54-debian-alternatives-for-external-tools.patch b/debian/patches/54-debian-alternatives-for-external-tools.patch new file mode 100644 index 00000000000..0031989718d --- /dev/null +++ b/debian/patches/54-debian-alternatives-for-external-tools.patch @@ -0,0 +1,28 @@ +Author: Martin Pitt +Description: Use Debian alternatives for external tools instead of hardcoded programs +Forwarded: No, Debian specific + +--- a/src/bin/psql/settings.h ++++ b/src/bin/psql/settings.h +@@ -19,8 +19,8 @@ + #define DEFAULT_EDITOR "notepad.exe" + /* no DEFAULT_EDITOR_LINENUMBER_ARG for Notepad */ + #else +-#define DEFAULT_EDITOR "vi" +-#define DEFAULT_EDITOR_LINENUMBER_ARG "+" ++#define DEFAULT_EDITOR "sensible-editor" ++/*#define DEFAULT_EDITOR_LINENUMBER_ARG "+"*/ + #endif + + #define DEFAULT_PROMPT1 "%/%R%x%# " +--- a/src/include/fe_utils/print.h ++++ b/src/include/fe_utils/print.h +@@ -20,7 +20,7 @@ + + /* This is not a particularly great place for this ... */ + #ifndef __CYGWIN__ +-#define DEFAULT_PAGER "more" ++#define DEFAULT_PAGER "pager" + #else + #define DEFAULT_PAGER "less" + #endif diff --git a/debian/patches/70-history b/debian/patches/70-history new file mode 100644 index 00000000000..34c868357f9 --- /dev/null +++ b/debian/patches/70-history @@ -0,0 +1,13 @@ +Author: Christoph Berg +Description: Document Debian location of release notes files. +Forwarded: No, Debian specific + +--- a/HISTORY ++++ b/HISTORY +@@ -3,3 +3,6 @@ + + Distribution file sets include release notes for their version and preceding + versions. Visit the file doc/src/sgml/html/release.html in an HTML browser. ++ ++On Debian systems, the release notes are contained in the postgresql-doc-* ++packages, located in /usr/share/doc/postgresql-doc-*/html/release.html. diff --git a/debian/patches/autoconf2.69 b/debian/patches/autoconf2.69 new file mode 100644 index 00000000000..429044e698b --- /dev/null +++ b/debian/patches/autoconf2.69 @@ -0,0 +1,7 @@ +--- a/configure.ac ++++ b/configure.ac +@@ -22,4 +21,0 @@ AC_INIT([PostgreSQL], [15devel], [pgsql- +-m4_if(m4_defn([m4_PACKAGE_VERSION]), [2.69], [], [m4_fatal([Autoconf version 2.69 is required. +-Untested combinations of 'autoconf' and PostgreSQL versions are not +-recommended. You can remove the check from 'configure.ac' but it is then +-your responsibility whether the result works or not.])]) diff --git a/debian/patches/filter-debug-prefix-map b/debian/patches/filter-debug-prefix-map new file mode 100644 index 00000000000..08f42b01bd4 --- /dev/null +++ b/debian/patches/filter-debug-prefix-map @@ -0,0 +1,34 @@ +To make the PostgreSQL server packages build reproducibly, we need to remove +the build path from -fdebug-prefix-map and -ffile-prefix-map in CFLAGS. + +* The actual server build still uses the original CFLAGS so the build path is + correctly mapped in the object files. +* The information printed by the pg_config binary and the system view is + filtered in src/common/Makefile and the configure script. +* The build paths stored in Makefile.global are filtered in debian/rules. + (abs_top_builddir, abs_top_srcdir, configure_args, CFLAGS) +* To make PGXS module builds reproducible, pg_buildext copies the environment + CFLAGS to COPT where Makefile.global picks them up, using the prefix maps + from dpkg-buildflags. + +--- a/src/common/Makefile ++++ b/src/common/Makefile +@@ -33,7 +33,7 @@ STD_CPPFLAGS := $(filter-out -I$(top_src + STD_LDFLAGS := $(filter-out -L$(top_builddir)/src/common -L$(top_builddir)/src/port,$(LDFLAGS)) + override CPPFLAGS += -DVAL_CC="\"$(CC)\"" + override CPPFLAGS += -DVAL_CPPFLAGS="\"$(STD_CPPFLAGS)\"" +-override CPPFLAGS += -DVAL_CFLAGS="\"$(CFLAGS)\"" ++override CPPFLAGS += -DVAL_CFLAGS="\"$(filter-out -fdebug-prefix-map=% -ffile-prefix-map=%,$(CFLAGS))\"" + override CPPFLAGS += -DVAL_CFLAGS_SL="\"$(CFLAGS_SL)\"" + override CPPFLAGS += -DVAL_LDFLAGS="\"$(STD_LDFLAGS)\"" + override CPPFLAGS += -DVAL_LDFLAGS_EX="\"$(LDFLAGS_EX)\"" +--- a/configure.ac ++++ b/configure.ac +@@ -27,6 +27,7 @@ AC_COPYRIGHT([Copyright (c) 1996-2023, P + AC_CONFIG_SRCDIR([src/backend/access/common/heaptuple.c]) + AC_CONFIG_AUX_DIR(config) + AC_PREFIX_DEFAULT(/usr/local/pgsql) ++[ac_configure_args=$(echo "$ac_configure_args" | sed -e "s/ -f\(debug\|file\)-prefix-map=[^' ]*//g")] + AC_DEFINE_UNQUOTED(CONFIGURE_ARGS, ["$ac_configure_args"], [Saved arguments from configure]) + + [PG_MAJORVERSION=`expr "$PACKAGE_VERSION" : '\([0-9][0-9]*\)'`] diff --git a/debian/patches/focal-arm64-outline-atomics b/debian/patches/focal-arm64-outline-atomics new file mode 100644 index 00000000000..b87d8f50eb9 --- /dev/null +++ b/debian/patches/focal-arm64-outline-atomics @@ -0,0 +1,23 @@ +Enable outline-atomics on arm64. + +The flag was added in focal's gcc, but is off by default there. It is enabled +by default on all later distributions (hirsute, impish, bullseye, bookwork, +sid). + +https://www.postgresql.org/message-id/flat/1635221042457.21654%40amazon.com + +This patch can be removed once focal is EOL. + +--- a/configure.ac ++++ b/configure.ac +@@ -576,6 +576,10 @@ if test "$GCC" = yes -a "$ICC" = no; the + if test -n "$NOT_THE_CFLAGS"; then + CFLAGS="$CFLAGS -Wno-cast-function-type-strict" + fi ++ if test x"$host_cpu" == x"aarch64"; then ++ PGAC_PROG_CC_CFLAGS_OPT([-moutline-atomics]) ++ PGAC_PROG_CXX_CFLAGS_OPT([-moutline-atomics]) ++ fi + elif test "$ICC" = yes; then + # Intel's compiler has a bug/misoptimization in checking for + # division by NAN (NaN == 0), -mp1 fixes it, so add it to the CFLAGS. diff --git a/debian/patches/hurd-iovec b/debian/patches/hurd-iovec new file mode 100644 index 00000000000..790909ae1f0 --- /dev/null +++ b/debian/patches/hurd-iovec @@ -0,0 +1,16 @@ +hurd-i386 does not define IOV_MAX + +--- a/src/include/port/pg_iovec.h ++++ b/src/include/port/pg_iovec.h +@@ -19,6 +19,11 @@ + #include /* IWYU pragma: export */ + #include + ++/* POSIX requires at least 16 as a maximum iovcnt. */ ++#ifndef IOV_MAX ++#define IOV_MAX 16 ++#endif ++ + #else + + /* POSIX requires at least 16 as a maximum iovcnt. */ diff --git a/debian/patches/jit-s390x b/debian/patches/jit-s390x new file mode 100644 index 00000000000..deb64e6af04 --- /dev/null +++ b/debian/patches/jit-s390x @@ -0,0 +1,96 @@ +From 0edaa982336823d4d7af8f10b91579fe0099ef3d Mon Sep 17 00:00:00 2001 +From: Tom Stellard +Date: Tue, 20 Apr 2021 20:14:21 -0700 +Subject: [PATCH] jit: Workaround potential datalayout mismatch on s390x + +LLVM's s390x target uses a different datalayout for z13 and newer processors. +If llvmjit_types.bc is compiled to target a processor older than z13, and +then the JIT runs on a z13 or newer processor, then there will be a mismatch +in datalayouts between llvmjit_types.bc and the JIT engine. This mismatch +causes the JIT to fail at runtime. +--- + src/backend/jit/llvm/llvmjit.c | 46 ++++++++++++++++++++++++++++++++-- + 1 file changed, 44 insertions(+), 2 deletions(-) + +--- a/src/backend/jit/llvm/llvmjit.c ++++ b/src/backend/jit/llvm/llvmjit.c +@@ -777,6 +777,37 @@ llvm_compile_module(LLVMJitContext *cont + } + + /* ++ * For the systemz target, LLVM uses a different datalayout for z13 and newer ++ * CPUs than it does for older CPUs. This can cause a mismatch in datalayouts ++ * in the case where the llvm_types_module is compiled with a pre-z13 CPU ++ * and the JIT is running on z13 or newer. ++ * See computeDataLayout() function in ++ * llvm/lib/Target/SystemZ/SystemZTargetMachine.cpp for information on the ++ * datalayout differences. ++ */ ++static bool ++needs_systemz_workaround(void) ++{ ++ bool ret = false; ++#ifdef __s390x__ ++ LLVMContextRef llvm_context; ++ LLVMTypeRef vec_type; ++ LLVMTargetDataRef llvm_layoutref; ++ if (strncmp(LLVMGetTargetName(llvm_targetref), "systemz", strlen("systemz"))) ++ { ++ return false; ++ } ++ ++ llvm_context = LLVMGetModuleContext(llvm_types_module); ++ vec_type = LLVMVectorType(LLVMIntTypeInContext(llvm_context, 32), 4); ++ llvm_layoutref = LLVMCreateTargetData(llvm_layout); ++ ret = (LLVMABIAlignmentOfType(llvm_layoutref, vec_type) == 16); ++ LLVMDisposeTargetData(llvm_layoutref); ++#endif ++ return ret; ++} ++ ++/* + * Per session initialization. + */ + static void +@@ -785,6 +816,7 @@ llvm_session_initialize(void) + MemoryContext oldcontext; + char *error = NULL; + char *cpu = NULL; ++ char *host_features = NULL; + char *features = NULL; + LLVMTargetMachineRef opt0_tm; + LLVMTargetMachineRef opt3_tm; +@@ -826,10 +858,17 @@ llvm_session_initialize(void) + * features not all CPUs have (weird, huh). + */ + cpu = LLVMGetHostCPUName(); +- features = LLVMGetHostCPUFeatures(); ++ features = host_features = LLVMGetHostCPUFeatures(); + elog(DEBUG2, "LLVMJIT detected CPU \"%s\", with features \"%s\"", + cpu, features); + ++ if (needs_systemz_workaround()) ++ { ++ const char *no_vector =",-vector"; ++ features = malloc(sizeof(char) * (strlen(host_features) + strlen(no_vector) + 1)); ++ sprintf(features, "%s%s", host_features, no_vector); ++ } ++ + opt0_tm = + LLVMCreateTargetMachine(llvm_targetref, llvm_triple, cpu, features, + LLVMCodeGenLevelNone, +@@ -843,8 +882,13 @@ llvm_session_initialize(void) + + LLVMDisposeMessage(cpu); + cpu = NULL; +- LLVMDisposeMessage(features); ++ if (features != host_features) ++ { ++ free(features); ++ } + features = NULL; ++ LLVMDisposeMessage(host_features); ++ host_features = NULL; + + /* force symbols in main binary to be loaded */ + LLVMLoadLibraryPermanently(NULL); diff --git a/debian/patches/libpgport-pkglibdir b/debian/patches/libpgport-pkglibdir new file mode 100644 index 00000000000..d0d3278eafe --- /dev/null +++ b/debian/patches/libpgport-pkglibdir @@ -0,0 +1,82 @@ +Author: Christoph Berg +Description: Move libpgport/libpgcommon/libpgfeutils from libdir to pkglibdir + This allows client applications to link to version-specific libraries. + Used by pg-checksums. +Forwarded: No, (somewhat) Debian specific + +--- a/src/common/Makefile ++++ b/src/common/Makefile +@@ -126,15 +126,15 @@ distprep: kwlist_d.h + + # libpgcommon is needed by some contrib + install: all installdirs +- $(INSTALL_STLIB) libpgcommon.a '$(DESTDIR)$(libdir)/libpgcommon.a' +- $(INSTALL_STLIB) libpgcommon_shlib.a '$(DESTDIR)$(libdir)/libpgcommon_shlib.a' ++ $(INSTALL_STLIB) libpgcommon.a '$(DESTDIR)$(pkglibdir)/libpgcommon.a' ++ $(INSTALL_STLIB) libpgcommon_shlib.a '$(DESTDIR)$(pkglibdir)/libpgcommon_shlib.a' + + installdirs: +- $(MKDIR_P) '$(DESTDIR)$(libdir)' ++ $(MKDIR_P) '$(DESTDIR)$(pkglibdir)' + + uninstall: +- rm -f '$(DESTDIR)$(libdir)/libpgcommon.a' +- rm -f '$(DESTDIR)$(libdir)/libpgcommon_shlib.a' ++ rm -f '$(DESTDIR)$(pkglibdir)/libpgcommon.a' ++ rm -f '$(DESTDIR)$(pkglibdir)/libpgcommon_shlib.a' + + libpgcommon.a: $(OBJS_FRONTEND) + rm -f $@ +--- a/src/fe_utils/Makefile ++++ b/src/fe_utils/Makefile +@@ -52,13 +52,13 @@ distprep: psqlscan.c + + # libpgfeutils could be useful to contrib, so install it + install: all installdirs +- $(INSTALL_STLIB) libpgfeutils.a '$(DESTDIR)$(libdir)/libpgfeutils.a' ++ $(INSTALL_STLIB) libpgfeutils.a '$(DESTDIR)$(pkglibdir)/libpgfeutils.a' + + installdirs: +- $(MKDIR_P) '$(DESTDIR)$(libdir)' ++ $(MKDIR_P) '$(DESTDIR)$(pkglibdir)' + + uninstall: +- rm -f '$(DESTDIR)$(libdir)/libpgfeutils.a' ++ rm -f '$(DESTDIR)$(pkglibdir)/libpgfeutils.a' + + clean distclean: + rm -f libpgfeutils.a $(OBJS) lex.backup +--- a/src/port/Makefile ++++ b/src/port/Makefile +@@ -70,15 +70,15 @@ all: libpgport.a libpgport_shlib.a libpg + + # libpgport is needed by some contrib + install: all installdirs +- $(INSTALL_STLIB) libpgport.a '$(DESTDIR)$(libdir)/libpgport.a' +- $(INSTALL_STLIB) libpgport_shlib.a '$(DESTDIR)$(libdir)/libpgport_shlib.a' ++ $(INSTALL_STLIB) libpgport.a '$(DESTDIR)$(pkglibdir)/libpgport.a' ++ $(INSTALL_STLIB) libpgport_shlib.a '$(DESTDIR)$(pkglibdir)/libpgport_shlib.a' + + installdirs: +- $(MKDIR_P) '$(DESTDIR)$(libdir)' ++ $(MKDIR_P) '$(DESTDIR)$(pkglibdir)' + + uninstall: +- rm -f '$(DESTDIR)$(libdir)/libpgport.a' +- rm -f '$(DESTDIR)$(libdir)/libpgport_shlib.a' ++ rm -f '$(DESTDIR)$(pkglibdir)/libpgport.a' ++ rm -f '$(DESTDIR)$(pkglibdir)/libpgport_shlib.a' + + libpgport.a: $(OBJS) + rm -f $@ +--- a/src/Makefile.global.in ++++ b/src/Makefile.global.in +@@ -608,7 +608,7 @@ libpq_pgport = $(libpq) + # done if they don't, since they will have satisfied all their references + # from these libraries.) + ifdef PGXS +-libpq_pgport_shlib = -L$(libdir) -lpgcommon_shlib -lpgport_shlib $(libpq) ++libpq_pgport_shlib = -L$(pkglibdir) -lpgcommon_shlib -lpgport_shlib $(libpq) + else + libpq_pgport_shlib = -L$(top_builddir)/src/common -lpgcommon_shlib -L$(top_builddir)/src/port -lpgport_shlib $(libpq) + endif diff --git a/debian/patches/move-pages32 b/debian/patches/move-pages32 new file mode 100644 index 00000000000..cf38e253c4e --- /dev/null +++ b/debian/patches/move-pages32 @@ -0,0 +1,49 @@ +Work around a Linux bug in move_pages + +In 32-bit mode on 64-bit kernels, move_pages() does not correctly advance to +the next chunk. Work around by not asking for more than 16 pages at once so +move_pages() internal loop is not executed more than once. + +https://www.postgresql.org/message-id/flat/a3a4fe3d-1a80-4e03-aa8e-150ee15f6c35%40vondra.me#6abe7eaa802b5b07bb70cc3229e63a9f +https://marc.info/?l=linux-mm&m=175077821909222&w=2 + +--- a/contrib/pg_buffercache/pg_buffercache_pages.c ++++ b/contrib/pg_buffercache/pg_buffercache_pages.c +@@ -390,8 +390,15 @@ pg_buffercache_numa_pages(PG_FUNCTION_AR + memset(os_page_status, 0xff, sizeof(int) * os_page_count); + + /* Query NUMA status for all the pointers */ +- if (pg_numa_query_pages(0, os_page_count, os_page_ptrs, os_page_status) == -1) +- elog(ERROR, "failed NUMA pages inquiry: %m"); ++#define NUMA_QUERY_CHUNK_SIZE 16 /* has to be <= DO_PAGES_STAT_CHUNK_NR (do_pages_stat())*/ ++ for (uint64 chunk_start = 0; chunk_start < os_page_count; chunk_start += NUMA_QUERY_CHUNK_SIZE) { ++ uint64 chunk_size = Min(NUMA_QUERY_CHUNK_SIZE, os_page_count - chunk_start); ++ ++ if (pg_numa_query_pages(0, chunk_size, &os_page_ptrs[chunk_start], ++ &os_page_status[chunk_start]) == -1) ++ elog(ERROR, "failed NUMA pages inquiry status: %m"); ++ } ++#undef NUMA_QUERY_CHUNK_SIZE + + /* Initialize the multi-call context, load entries about buffers */ + +--- a/src/backend/storage/ipc/shmem.c ++++ b/src/backend/storage/ipc/shmem.c +@@ -689,8 +689,15 @@ pg_get_shmem_allocations_numa(PG_FUNCTIO + CHECK_FOR_INTERRUPTS(); + } + +- if (pg_numa_query_pages(0, shm_ent_page_count, page_ptrs, pages_status) == -1) +- elog(ERROR, "failed NUMA pages inquiry status: %m"); ++#define NUMA_QUERY_CHUNK_SIZE 16 /* has to be <= DO_PAGES_STAT_CHUNK_NR (do_pages_stat())*/ ++ for (uint64 chunk_start = 0; chunk_start < shm_ent_page_count; chunk_start += NUMA_QUERY_CHUNK_SIZE) { ++ uint64 chunk_size = Min(NUMA_QUERY_CHUNK_SIZE, shm_ent_page_count - chunk_start); ++ ++ if (pg_numa_query_pages(0, chunk_size, &page_ptrs[chunk_start], ++ &pages_status[chunk_start]) == -1) ++ elog(ERROR, "failed NUMA pages inquiry status: %m"); ++ } ++#undef NUMA_QUERY_CHUNK_SIZE + + /* Count number of NUMA nodes used for this shared memory entry */ + memset(nodes, 0, sizeof(Size) * (max_nodes + 1)); diff --git a/debian/patches/pgstat-report-conflicts-immediately.patch b/debian/patches/pgstat-report-conflicts-immediately.patch new file mode 100644 index 00000000000..57603107f76 --- /dev/null +++ b/debian/patches/pgstat-report-conflicts-immediately.patch @@ -0,0 +1,37 @@ +diff --git i/src/backend/utils/activity/pgstat_database.c w/src/backend/utils/activity/pgstat_database.c +index 7149f22f729..bb36d73ec04 100644 +--- i/src/backend/utils/activity/pgstat_database.c ++++ w/src/backend/utils/activity/pgstat_database.c +@@ -81,12 +81,22 @@ void + pgstat_report_recovery_conflict(int reason) + { + PgStat_StatDBEntry *dbentry; ++ PgStat_EntryRef *entry_ref; ++ PgStatShared_Database *sharedent; + + Assert(IsUnderPostmaster); + if (!pgstat_track_counts) + return; + +- dbentry = pgstat_prep_database_pending(MyDatabaseId); ++ /* ++ * Update the shared stats directly - recovery conflicts should never be ++ * common enough for that to be a problem. ++ */ ++ entry_ref = ++ pgstat_get_entry_ref_locked(PGSTAT_KIND_DATABASE, MyDatabaseId, InvalidOid, false); ++ ++ sharedent = (PgStatShared_Database *) entry_ref->shared_stats; ++ dbentry = &sharedent->stats; + + switch (reason) + { +@@ -116,6 +126,8 @@ pgstat_report_recovery_conflict(int reason) + dbentry->conflict_startup_deadlock++; + break; + } ++ ++ pgstat_unlock_entry(entry_ref); + } + + /* diff --git a/debian/patches/series b/debian/patches/series new file mode 100644 index 00000000000..0445628ec33 --- /dev/null +++ b/debian/patches/series @@ -0,0 +1,14 @@ +50-per-version-dirs.patch +51-default-sockets-in-var.patch +52-tutorial-README.patch +53-pg_service.conf_directory_doc.patch +54-debian-alternatives-for-external-tools.patch +70-history +filter-debug-prefix-map +libpgport-pkglibdir +autoconf2.69 +focal-arm64-outline-atomics +jit-s390x +hurd-iovec +pgstat-report-conflicts-immediately.patch +move-pages32 diff --git a/debian/po/POTFILES.in b/debian/po/POTFILES.in new file mode 100644 index 00000000000..6fd71790bdd --- /dev/null +++ b/debian/po/POTFILES.in @@ -0,0 +1 @@ +[type: gettext/rfc822deb] postgresql-18.templates diff --git a/debian/po/ca.po b/debian/po/ca.po new file mode 100644 index 00000000000..7e163ef6341 --- /dev/null +++ b/debian/po/ca.po @@ -0,0 +1,38 @@ +# Catalan translation of postgresql-17's debconf messages +# Copyright © 2024 Free Software Foundation, Inc. +# This file is distributed under the same license as the postgresql-17 package. +# poc senderi , 2024. +# +msgid "" +msgstr "" +"Project-Id-Version: postgresql-11\n" +"Report-Msgid-Bugs-To: postgresql-11@packages.debian.org\n" +"POT-Creation-Date: 2019-01-09 15:22+0100\n" +"PO-Revision-Date: 2024-11-11 22:07+0100\n" +"Language: ca\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Last-Translator: poc senderi \n" +"Language-Team: Catalan \n" +"X-Generator: Poedit 2.4.2\n" + +#. Type: boolean +#. Description +#: ../postgresql-11.templates:1001 +msgid "Remove PostgreSQL directories when package is purged?" +msgstr "" +"Voleu esborrar els directoris del PostgreSQL quan el paquet sigui purgat?" + +#. Type: boolean +#. Description +#: ../postgresql-11.templates:1001 +msgid "" +"Removing the PostgreSQL server package will leave existing database clusters " +"intact, i.e. their configuration, data, and log directories will not be " +"removed. On purging the package, the directories can optionally be removed." +msgstr "" +"A l'esborrar el paquet del servidor PostgreSQL romandran les bases de dades " +"intactes, això és, la configuració, les dades i els directoris de registres " +"no seran esborrats. Al purgar el paquet, aquests directoris poden ser " +"opcionalment també esborrats." diff --git a/debian/po/de.po b/debian/po/de.po new file mode 100644 index 00000000000..df2bd4d7012 --- /dev/null +++ b/debian/po/de.po @@ -0,0 +1,37 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER +# This file is distributed under the same license as the postgresql-11 package. +# Copyright (C) Helge Kreutzmann , 2019. +# +msgid "" +msgstr "" +"Project-Id-Version: postgresql-11 11.1-2\n" +"Report-Msgid-Bugs-To: postgresql-11@packages.debian.org\n" +"POT-Creation-Date: 2019-01-09 15:22+0100\n" +"PO-Revision-Date: 2019-01-19 07:33+0100\n" +"Last-Translator: Helge Kreutzmann \n" +"Language-Team: German \n" +"Language: de\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" + +#. Type: boolean +#. Description +#: ../postgresql-11.templates:1001 +msgid "Remove PostgreSQL directories when package is purged?" +msgstr "" +"PostgreSQL-Verzeichnisse entfernen, wenn das Paket endgültig gelöscht wird?" + +#. Type: boolean +#. Description +#: ../postgresql-11.templates:1001 +msgid "" +"Removing the PostgreSQL server package will leave existing database clusters " +"intact, i.e. their configuration, data, and log directories will not be " +"removed. On purging the package, the directories can optionally be removed." +msgstr "" +"Beim Entfernen der PostgreSQL-Server-Pakete werden existierende Datenbank-" +"Cluster intakt gelassen, d.h. ihre Konfigurations-, Daten- und " +"Log-Verzeichnisse werden nicht entfernt. Beim endgültigen Löschen des " +"Pakets können die Verzeichnisse optional entfernt werden." diff --git a/debian/po/es.po b/debian/po/es.po new file mode 100644 index 00000000000..d408e656a0d --- /dev/null +++ b/debian/po/es.po @@ -0,0 +1,58 @@ +# postgresql-13 po-debconf translation to Spanish. +# Copyright (C) 2021 Software in the Public Interest +# This file is distributed under the same license as the postgresql-13 package. +# +# Changes: +# - Initial translation +# Jonathan Bustillos , 2021. +# +# Traductores, si no conocen el formato PO, merece la pena leer la +# documentación de gettext, especialmente las secciones dedicadas a este +# formato, por ejemplo ejecutando: +# info -n '(gettext)PO Files' +# info -n '(gettext)Header Entry' +# +# Equipo de traducción al español, por favor lean antes de traducir +# los siguientes documentos: +# +# - El proyecto de traducción de Debian al español +# http://www.debian.org/intl/spanish/ +# especialmente las notas y normas de traducción en +# http://www.debian.org/intl/spanish/notas +# +# - La guía de traducción de po's de debconf: +# /usr/share/doc/po-debconf/README-trans +# o http://www.debian.org/intl/l10n/po-debconf/README-trans +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2021-03-31 18:37+0000\n" +"PO-Revision-Date: 2021-04-03 14:25-0600\n" +"Last-Translator: Jonathan Bustillos \n" +"Language-Team: Debian Spanish \n" +"Language: \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Plural-Forms: nplurals=2; plural=(n != 1);\n" +"X-Generator: Gtranslator 2.91.7\n" + +#. Type: boolean +#. Description +#: ../postgresql-13.templates:1001 +msgid "Remove PostgreSQL directories when package is purged?" +msgstr "¿Eliminar los directorios de PostgreSQL cuando se purga el paquete?" + +#. Type: boolean +#. Description +#: ../postgresql-13.templates:1001 +msgid "" +"Removing the PostgreSQL server package will leave existing database clusters " +"intact, i.e. their configuration, data, and log directories will not be " +"removed. On purging the package, the directories can optionally be removed." +msgstr "" +"La eliminación del paquete del servidor PostgreSQL dejará intactos los " +"clusters de bases de datos existentes, es decir, no se eliminarán sus " +"directorios de configuración, datos y registro. Al purgar el paquete, los " +"directorios pueden ser eliminados opcionalmente." diff --git a/debian/po/fr.po b/debian/po/fr.po new file mode 100644 index 00000000000..cc3c6ee55c6 --- /dev/null +++ b/debian/po/fr.po @@ -0,0 +1,39 @@ +# Translation of postgresql debconf templates to French +# Copyright (C) 2019 Debian French l10n team +# This file is distributed under the same license as the postgresql-11 package. +# +# Jean-Pierre Giraud , 2019. +msgid "" +msgstr "" +"Project-Id-Version: postgresql-11\n" +"Report-Msgid-Bugs-To: postgresql-11@packages.debian.org\n" +"POT-Creation-Date: 2019-01-09 15:22+0100\n" +"PO-Revision-Date: 2019-01-14 14:25+0100\n" +"Last-Translator: Jean-Pierre Giraud \n" +"Language-Team: French \n" +"Language: fr\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Plural-Forms: nplurals=2; plural=(n > 1);\n" +"X-Generator: Lokalize 2.0\n" + +#. Type: boolean +#. Description +#: ../postgresql-11.templates:1001 +msgid "Remove PostgreSQL directories when package is purged?" +msgstr "" +"Faut-il supprimer les répertoires de PostgreSQL lors de la purge du paquet ?" + +#. Type: boolean +#. Description +#: ../postgresql-11.templates:1001 +msgid "" +"Removing the PostgreSQL server package will leave existing database clusters " +"intact, i.e. their configuration, data, and log directories will not be " +"removed. On purging the package, the directories can optionally be removed." +msgstr "" +"La suppression du paquet du serveur PostgreSQL laissera les grappes de bases " +"de données existantes intactes, c'est-à-dire que leurs répertoires de " +"configuration, de données et de journal ne seront pas supprimés. Lors de la " +"purge du paquet, les répertoires peuvent être supprimés de façon optionnelle." diff --git a/debian/po/it.po b/debian/po/it.po new file mode 100644 index 00000000000..d8020b9fd8b --- /dev/null +++ b/debian/po/it.po @@ -0,0 +1,37 @@ +# postgresql-14 Italian translation. +# Copyright (C) 2022 postgresql-14's copyright holder +# This file is distributed under the same license as the postgresql-14 package. +# Ceppo , 2022. +# +msgid "" +msgstr "" +"Project-Id-Version: postgresql-14\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2022-08-23 19:57+0000\n" +"PO-Revision-Date: 2022-08-23 00:00+0000\n" +"Last-Translator: Ceppo \n" +"Language-Team: Italian \n" +"Language: it\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" + +#. Type: boolean +#. Description +#: ../postgresql-14.templates:1001 +msgid "Remove PostgreSQL directories when package is purged?" +msgstr "Rimuovere le directory di PostgreSQL quando viene eseguito il purge " +"del pacchetto?" + +#. Type: boolean +#. Description +#: ../postgresql-14.templates:1001 +msgid "" +"Removing the PostgreSQL server package will leave existing database clusters " +"intact, i.e. their configuration, data, and log directories will not be " +"removed. On purging the package, the directories can optionally be removed." +msgstr "" +"La rimozione del pacchetto server di PostgreSQL lascerà intatti i cluster " +"di database esistenti, cioè i loro dati, configurazione e directory di log " +"non saranno rimossi. Eseguendo il purge del pacchetto, le directory possono " +"opzionalmente essere rimosse." diff --git a/debian/po/nl.po b/debian/po/nl.po new file mode 100644 index 00000000000..11cda877b96 --- /dev/null +++ b/debian/po/nl.po @@ -0,0 +1,40 @@ +# Dutch translation of postgresql-11 debconf templates. +# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER +# This file is distributed under the same license as the postgresql-11 package. +# FIRST AUTHOR , YEAR. +# Frans Spiesschaert , 2019. +# +msgid "" +msgstr "" +"Project-Id-Version: postgresql-11_11.1-2\n" +"Report-Msgid-Bugs-To: postgresql-11@packages.debian.org\n" +"POT-Creation-Date: 2019-01-09 15:22+0100\n" +"PO-Revision-Date: 2019-01-19 10:43+0100\n" +"Last-Translator: Frans Spiesschaert \n" +"Language-Team: Debian Dutch l10n Team \n" +"Language: nl\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Plural-Forms: nplurals=2; plural=(n != 1);\n" +"X-Generator: Gtranslator 2.91.7\n" + +#. Type: boolean +#. Description +#: ../postgresql-11.templates:1001 +msgid "Remove PostgreSQL directories when package is purged?" +msgstr "" +"De PostgreSQL-mappen verwijderen wanneer het pakket gewist (purged) wordt?" + +#. Type: boolean +#. Description +#: ../postgresql-11.templates:1001 +msgid "" +"Removing the PostgreSQL server package will leave existing database clusters " +"intact, i.e. their configuration, data, and log directories will not be " +"removed. On purging the package, the directories can optionally be removed." +msgstr "" +"Bij het verwijderen van het serverpakket van PostgreSQL blijven de bestaande " +"databaseclusters intact. Dit wil zeggen dat hun configuratie-, gegevens- en " +"logboekmappen niet verwijderd worden. Bij het wissen (purge) van het pakket, " +"kunnen de mappen naar keuze verwijderd worden." diff --git a/debian/po/pt.po b/debian/po/pt.po new file mode 100644 index 00000000000..ee294d15a44 --- /dev/null +++ b/debian/po/pt.po @@ -0,0 +1,39 @@ +# Translation of postgresql-11's debconf messages to European Portuguese +# Copyright (C) 2019 THE postgresql-11'S COPYRIGHT HOLDER +# This file is distributed under the same license as the postgresql-11 package. +# +# Américo Monteiro , 2019. +msgid "" +msgstr "" +"Project-Id-Version: postgresql-11 11.1-2\n" +"Report-Msgid-Bugs-To: postgresql-11@packages.debian.org\n" +"POT-Creation-Date: 2019-01-09 15:22+0100\n" +"PO-Revision-Date: 2019-01-15 00:34+0000\n" +"Last-Translator: Américo Monteiro \n" +"Language-Team: Portuguese <>\n" +"Language: pt\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Plural-Forms: nplurals=2; plural=(n != 1);\n" +"X-Generator: Lokalize 2.0\n" + +#. Type: boolean +#. Description +#: ../postgresql-11.templates:1001 +msgid "Remove PostgreSQL directories when package is purged?" +msgstr "Remover os directórios do PostgreSQL quando o pacote for purgado?" + +#. Type: boolean +#. Description +#: ../postgresql-11.templates:1001 +msgid "" +"Removing the PostgreSQL server package will leave existing database clusters " +"intact, i.e. their configuration, data, and log directories will not be " +"removed. On purging the package, the directories can optionally be removed." +msgstr "" +"Remover o pacote do servidor PostgreSQL irá deixar intactos agrupamentos de " +"bases de dados existentes, isto é, a sua configuração, dados, e relatórios " +"são serão removidos. Ao purgar o pacote, estes directórios podem " +"opcionalmente ser removidos." + diff --git a/debian/po/pt_BR.po b/debian/po/pt_BR.po new file mode 100644 index 00000000000..5e746cf1ca2 --- /dev/null +++ b/debian/po/pt_BR.po @@ -0,0 +1,37 @@ +# Debconf translations for postgresql-11. +# Copyright (C) 2019 THE postgresql-11'S COPYRIGHT HOLDER +# This file is distributed under the same license as the postgresql-11 package. +# Adriano Rafael Gomes , 2019. +# +msgid "" +msgstr "" +"Project-Id-Version: postgresql-11\n" +"Report-Msgid-Bugs-To: postgresql-11@packages.debian.org\n" +"POT-Creation-Date: 2019-01-09 15:22+0100\n" +"PO-Revision-Date: 2019-01-19 18:06-0200\n" +"Last-Translator: Adriano Rafael Gomes \n" +"Language-Team: Brazilian Portuguese \n" +"Language: pt_BR\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" + +#. Type: boolean +#. Description +#: ../postgresql-11.templates:1001 +msgid "Remove PostgreSQL directories when package is purged?" +msgstr "Remover diretórios do PostgreSQL ao expurgar o pacote?" + +#. Type: boolean +#. Description +#: ../postgresql-11.templates:1001 +msgid "" +"Removing the PostgreSQL server package will leave existing database clusters " +"intact, i.e. their configuration, data, and log directories will not be " +"removed. On purging the package, the directories can optionally be removed." +msgstr "" +"Remover o pacote do servidor PostgreSQL deixará os \"clusters\" de bancos de " +"dados existentes intactos, ou seja, suas configurações, dados e diretórios " +"de log não serão removidos. Ao expurgar o pacote, os diretórios podem ser " +"opcionalmente removidos." diff --git a/debian/po/ro.po b/debian/po/ro.po new file mode 100644 index 00000000000..f38fc5581b6 --- /dev/null +++ b/debian/po/ro.po @@ -0,0 +1,50 @@ +# Mesajele în limba română pentru pachetul postgresql. +# translation of postgresql-xx_ro.po to Romanian +# Copyright © 2023 THE PACKAGE'S COPYRIGHT HOLDER +# This file is distributed under the same license as the postgresql package. +# +# Remus-Gabriel Chelu , 2023. +# +# Cronologia traducerii fișierului „postgresql”: +# Traducerea inițială, făcută de R-GC, pentru versiunea postgresql-15_15.1-1. +# Actualizare a traducerii pentru versiunea Y, făcută de X, Y(anul). +# +msgid "" +msgstr "" +"Project-Id-Version: postgresql-15 15.1-1\n" +"Report-Msgid-Bugs-To: postgresql-15@packages.debian.org\n" +"POT-Creation-Date: 2023-01-19 20:42+0000\n" +"PO-Revision-Date: 2023-02-12 17:42+0100\n" +"Last-Translator: Remus-Gabriel Chelu \n" +"Language-Team: Romanian \n" +"Language: ro\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Plural-Forms: nplurals=3; plural=(n==1 ? 0 : n==0 || (n!=1 && n%100>=1 && " +"n%100<=19) ? 1 : 2);\n" +"X-Generator: Poedit 3.2.2\n" + +#. Type: boolean +#. Description +#: ../postgresql-15.templates:1001 +msgid "Remove PostgreSQL directories when package is purged?" +msgstr "Eliminați directoarele PostgreSQL atunci când pachetul este înlăturat?" + +# R-GC, scrie: +# la sugestia lui DȘ, am modificat traducerea +# acestui mesaj, de la: +# „... va lăsa intacte clusterele de baze de date existente, ...”, la: +# „... va lăsa intacte grupurile de servere (clusters) de baze de date existente, ...” +#. Type: boolean +#. Description +#: ../postgresql-15.templates:1001 +msgid "" +"Removing the PostgreSQL server package will leave existing database clusters " +"intact, i.e. their configuration, data, and log directories will not be " +"removed. On purging the package, the directories can optionally be removed." +msgstr "" +"Eliminarea pachetului de server PostgreSQL va lăsa intacte grupurile de servere " +"(clusters) de baze de date existente, adică configurația, datele și " +"directoarele lor de jurnal nu vor fi eliminate. La înlăturarea pachetului, " +"directoarele pot fi eliminate opțional." diff --git a/debian/po/ru.po b/debian/po/ru.po new file mode 100644 index 00000000000..9828439c667 --- /dev/null +++ b/debian/po/ru.po @@ -0,0 +1,39 @@ +# Russian translation of debconf template for postgresql-11 +# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER +# This file is distributed under the same license as the postgresql-11 package. +# Lev Lamberov , 2019 +# +msgid "" +msgstr "" +"Project-Id-Version: postgresql-11\n" +"Report-Msgid-Bugs-To: postgresql-11@packages.debian.org\n" +"POT-Creation-Date: 2019-01-09 15:22+0100\n" +"PO-Revision-Date: 2019-01-27 14:56+0500\n" +"Language-Team: Debian L10N Russian \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"X-Generator: Poedit 2.2.1\n" +"Last-Translator: Lev Lamberov \n" +"Plural-Forms: nplurals=3; plural=(n%10==1 && n%100!=11 ? 0 : n%10>=2 && n" +"%10<=4 && (n%100<12 || n%100>14) ? 1 : 2);\n" +"Language: ru\n" + +#. Type: boolean +#. Description +#: ../postgresql-11.templates:1001 +msgid "Remove PostgreSQL directories when package is purged?" +msgstr "Удалить каталоги PostgreSQL при вычищении пакета?" + +#. Type: boolean +#. Description +#: ../postgresql-11.templates:1001 +msgid "" +"Removing the PostgreSQL server package will leave existing database clusters " +"intact, i.e. their configuration, data, and log directories will not be " +"removed. On purging the package, the directories can optionally be removed." +msgstr "" +"При удалении серверного пакета PostgreSQL существующие кластеры баз данных " +"останутся нетронутыми. То есть, их каталоги с настройками, данными и " +"журналами не будут удалены. При вычистке пакета эти каталоги могут быть при " +"необходимости удалены." diff --git a/debian/po/sv.po b/debian/po/sv.po new file mode 100644 index 00000000000..0f9120fff98 --- /dev/null +++ b/debian/po/sv.po @@ -0,0 +1,36 @@ +# Translation of postgresql-16 debconf template to Swedish +# Copyright (C) 2023 Martin Bagge +# This file is distributed under the same license as the postgresql-16 package. +# +# Martin Bagge , 2023 +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"Report-Msgid-Bugs-To: postgresql-16@packages.debian.org\n" +"POT-Creation-Date: 2023-12-21 17:22+0100\n" +"PO-Revision-Date: 2023-12-20 22:17+0100\n" +"Last-Translator: Martin Bagge \n" +"Language-Team: Swedish \n" +"Language: sv\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" + +#. Type: boolean +#. Description +#: ../postgresql-16.templates:1001 +msgid "Remove PostgreSQL directories when package is purged?" +msgstr "Ska sökvägar för PostgreSQL tas bort när paketet rensas bort helt?" + +#. Type: boolean +#. Description +#: ../postgresql-16.templates:1001 +msgid "" +"Removing the PostgreSQL server package will leave existing database clusters " +"intact, i.e. their configuration, data, and log directories will not be " +"removed. On purging the package, the directories can optionally be removed." +msgstr "" +"När serverpaketet för PostgreSQL tas bort lämnas existerande databaskluster " +"kvar i sin helhet - det vill säga inställnings-, data- och logg-kataloger " +"kommer inte tas bort. När ett paket rensas bort helt (purge) kan katalogerna " +"tas bort automatiskt om så önskas." diff --git a/debian/po/tr.po b/debian/po/tr.po new file mode 100644 index 00000000000..e0bc253e234 --- /dev/null +++ b/debian/po/tr.po @@ -0,0 +1,41 @@ +# Turkish debconf translation of postgresql +# Copyright (C) 2025 Debian Turkish L10n Team +# This file is distributed under the same license as the postgresql package. +# +# Translators: +# Atila KOÇ , 2025. +# +msgid "" +msgstr "" +"Project-Id-Version: postgresql 17\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2025-04-29 17:06+0000\n" +"PO-Revision-Date: 2025-05-18 11:57+0300\n" +"Last-Translator: Atila KOÇ \n" +"Language-Team: Turkish \n" +"Language: tr\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Plural-Forms: nplurals=2; plural=(n > 1);\n" +"X-Generator: Poedit 3.6\n" + +#. Type: boolean +#. Description +#: ../postgresql-17.templates:1001 +msgid "Remove PostgreSQL directories when package is purged?" +msgstr "" +"PostgreSQL paketi temizlenerek kaldırıldığında, dizinleri de silinsin mi?" + +#. Type: boolean +#. Description +#: ../postgresql-17.templates:1001 +msgid "" +"Removing the PostgreSQL server package will leave existing database clusters " +"intact, i.e. their configuration, data, and log directories will not be " +"removed. On purging the package, the directories can optionally be removed." +msgstr "" +"PostgreSQL sunucu paketi kaldırıldığında varolan veritabanı kümelerini " +"öylece bırakır; örneğin onların yapılandırma dosyaları, verileri ve kayıt " +"dizinleri silinmez. Paket temizlenerek kaldırıldığında ise, bu dizinler " +"isteğe bağlı olarak silinebilir." diff --git a/debian/postgresql-18-jit.install b/debian/postgresql-18-jit.install new file mode 100644 index 00000000000..ea65125f165 --- /dev/null +++ b/debian/postgresql-18-jit.install @@ -0,0 +1,3 @@ +usr/lib/postgresql/*/lib/bitcode +usr/lib/postgresql/*/lib/llvmjit.so +usr/lib/postgresql/*/lib/llvmjit_types.bc diff --git a/debian/postgresql-18.install b/debian/postgresql-18.install new file mode 100755 index 00000000000..a3bac19dc4e --- /dev/null +++ b/debian/postgresql-18.install @@ -0,0 +1,60 @@ +#!/usr/bin/dh-exec + +usr/lib/postgresql/*/bin/initdb +usr/lib/postgresql/*/bin/oid2name +usr/lib/postgresql/*/bin/pg_archivecleanup +usr/lib/postgresql/*/bin/pgbench +usr/lib/postgresql/*/bin/pg_checksums +usr/lib/postgresql/*/bin/pg_controldata +usr/lib/postgresql/*/bin/pg_createsubscriber +usr/lib/postgresql/*/bin/pg_ctl +usr/lib/postgresql/*/bin/pg_resetwal +usr/lib/postgresql/*/bin/pg_rewind +usr/lib/postgresql/*/bin/pg_test_fsync +usr/lib/postgresql/*/bin/pg_test_timing +usr/lib/postgresql/*/bin/pg_upgrade +usr/lib/postgresql/*/bin/pg_waldump +usr/lib/postgresql/*/bin/pg_walsummary +usr/lib/postgresql/*/bin/postgres +usr/lib/postgresql/*/bin/vacuumlo +usr/lib/postgresql/*/lib/*.so +usr/share/locale/*/LC_MESSAGES/initdb-*.mo +usr/share/locale/*/LC_MESSAGES/pg_archivecleanup-*.mo +usr/share/locale/*/LC_MESSAGES/pg_checksums-*.mo +usr/share/locale/*/LC_MESSAGES/pg_controldata-*.mo +usr/share/locale/*/LC_MESSAGES/pg_ctl-*.mo +usr/share/locale/*/LC_MESSAGES/pg_resetwal-*.mo +usr/share/locale/*/LC_MESSAGES/pg_rewind-*.mo +usr/share/locale/*/LC_MESSAGES/pg_test_fsync-*.mo +usr/share/locale/*/LC_MESSAGES/pg_test_timing-*.mo +usr/share/locale/*/LC_MESSAGES/pg_upgrade-*.mo +usr/share/locale/*/LC_MESSAGES/pg_waldump-*.mo +usr/share/locale/*/LC_MESSAGES/pg_walsummary-*.mo +usr/share/locale/*/LC_MESSAGES/postgres-*.mo +usr/share/locale/*/LC_MESSAGES/plpgsql-*.mo +[linux-any] usr/share/postgresql/*/contrib/sepgsql.sql +usr/share/postgresql/*/errcodes.txt +usr/share/postgresql/*/extension/* +usr/share/postgresql/*/man/man1/initdb.1* +usr/share/postgresql/*/man/man1/oid2name.1* +usr/share/postgresql/*/man/man1/pg_archivecleanup.1* +usr/share/postgresql/*/man/man1/pgbench.1* +usr/share/postgresql/*/man/man1/pg_checksums.1* +usr/share/postgresql/*/man/man1/pg_controldata.1* +usr/share/postgresql/*/man/man1/pg_createsubscriber.1* +usr/share/postgresql/*/man/man1/pg_ctl.1* +usr/share/postgresql/*/man/man1/pg_resetwal.1* +usr/share/postgresql/*/man/man1/pg_rewind.1* +usr/share/postgresql/*/man/man1/pg_test_fsync.1* +usr/share/postgresql/*/man/man1/pg_test_timing.1* +usr/share/postgresql/*/man/man1/pg_upgrade.1* +usr/share/postgresql/*/man/man1/pg_waldump.1* +usr/share/postgresql/*/man/man1/pg_walsummary.1* +usr/share/postgresql/*/man/man1/postgres.1* +usr/share/postgresql/*/man/man1/vacuumlo.1* +usr/share/postgresql/*/timezonesets/* +usr/share/postgresql/*/tsearch_data +usr/share/postgresql/*/*.sql +usr/share/postgresql/*/*.conf.sample +usr/share/postgresql/*/postgres.bki +usr/share/postgresql/*/sql_features.txt diff --git a/debian/postgresql-18.lintian-overrides b/debian/postgresql-18.lintian-overrides new file mode 100644 index 00000000000..db5908332f9 --- /dev/null +++ b/debian/postgresql-18.lintian-overrides @@ -0,0 +1,20 @@ +# We test for /usr/bin/pg_dropcluster, but run it without path +command-with-path-in-maintainer-script + +# The World's Most Advanced Open Source Relational Database +description-synopsis-starts-with-article + +# We ship binaries and libs in subdirs of /usr/lib/postgresql +executable-in-usr-lib +repeated-path-segment lib * + +# These are PostgreSQL server plugins; some need no external libraries +hardening-no-fortify-functions [usr/lib/postgresql/*/lib/*] +library-not-linked-against-libc [usr/lib/postgresql/*/lib/*] +shared-library-lacks-prerequisites [usr/lib/postgresql/*/lib/*] + +# We use debconf in postrm only +no-debconf-config + +# We store the PostgreSQL catalog version in a custom control field +unknown-field *Postgresql-Catversion diff --git a/debian/postgresql-18.postinst b/debian/postgresql-18.postinst new file mode 100644 index 00000000000..be698435c3f --- /dev/null +++ b/debian/postgresql-18.postinst @@ -0,0 +1,13 @@ +#!/bin/sh + +set -e + +VERSION=${DPKG_MAINTSCRIPT_PACKAGE##*-} + +if [ "$1" = configure ]; then + . /usr/share/postgresql-common/maintscripts-functions + + configure_version $VERSION "$2" +fi + +#DEBHELPER# diff --git a/debian/postgresql-18.postrm b/debian/postgresql-18.postrm new file mode 100644 index 00000000000..c9f88934327 --- /dev/null +++ b/debian/postgresql-18.postrm @@ -0,0 +1,80 @@ +#!/bin/sh + +set -e + +VERSION=${DPKG_MAINTSCRIPT_PACKAGE##*-} + +clean_dir() { + if [ -d "$1" ] && [ ! -L "$1" ]; then + rmdir "$1" >/dev/null 2>/dev/null || true + fi +} + +drop_cluster() { + # if we still have the postgresql-common package, use it to also shutdown + # server, etc.; otherwise just remove the directories + if [ -x /usr/bin/pg_dropcluster ]; then + pg_dropcluster --stop-server $VERSION "$1" + else + # remove data directory + PGDATALINK="/etc/postgresql/$VERSION/$1/pgdata" + if [ -e "$PGDATALINK" ]; then + rm -rf $(readlink -f "$PGDATALINK") "$PGDATALINK" + else + rm -rf "/var/lib/postgresql/$VERSION/$1/" + fi + + # remove log file, including rotated ones + LOGLINK="/etc/postgresql/$VERSION/$1/log" + if [ -e "$LOGLINK" ]; then + LOG=$(readlink -f "$LOGLINK") + rm -f $LOG* "$LOGLINK" + else + rm -f /var/log/postgresql/postgresql-$VERSION-"$1".log* + fi + + # remove conffiles + for f in pg_hba.conf pg_ident.conf postgresql.conf start.conf environment pg_ctl.conf; do + rm -f /etc/postgresql/$VERSION/"$1"/$f + done + # remove empty conf.d directories + for d in /etc/postgresql/$VERSION/"$1"/*/; do + clean_dir "$d" + done + + clean_dir /etc/postgresql/$VERSION/"$1" + fi +} + +purge_package () { + # ask the user if they want to remove clusters. If debconf is not + # available, just remove everything + if [ -e /usr/share/debconf/confmodule ]; then + db_set $DPKG_MAINTSCRIPT_PACKAGE/postrm_purge_data true + db_input high $DPKG_MAINTSCRIPT_PACKAGE/postrm_purge_data || : + db_go || : + db_get $DPKG_MAINTSCRIPT_PACKAGE/postrm_purge_data || : + [ "$RET" = "false" ] && return 0 + fi + + for c in /etc/postgresql/$VERSION/*; do + [ -e "$c/postgresql.conf" ] || continue + cluster=$(basename "$c") + echo "Dropping cluster $cluster..." + drop_cluster "$cluster" + done + + clean_dir /etc/postgresql/$VERSION + clean_dir /var/lib/postgresql/$VERSION + clean_dir /var/log/postgresql/$VERSION +} + +if [ "$1" = purge ] && [ -d "/etc/postgresql/$VERSION" ] && [ "$(ls /etc/postgresql/$VERSION)" ]; then + # can't load debconf from a function + if [ -e /usr/share/debconf/confmodule ]; then + . /usr/share/debconf/confmodule + fi + purge_package +fi + +#DEBHELPER# diff --git a/debian/postgresql-18.preinst b/debian/postgresql-18.preinst new file mode 100644 index 00000000000..a1bccbc9191 --- /dev/null +++ b/debian/postgresql-18.preinst @@ -0,0 +1,18 @@ +#!/bin/sh + +set -e + +MAJOR_VER="${DPKG_MAINTSCRIPT_PACKAGE#postgresql-}" +CATVERSION="@CATVERSION@" # set by override_dh_installdeb + +case $1 in + install|upgrade) + if [ "$2" ]; then + . /usr/share/postgresql-common/maintscripts-functions + preinst_check_catversion "$MAJOR_VER" "$CATVERSION" + fi ;; +esac + +#DEBHELPER# + +exit 0 diff --git a/debian/postgresql-18.prerm b/debian/postgresql-18.prerm new file mode 100644 index 00000000000..828c6fbe9f5 --- /dev/null +++ b/debian/postgresql-18.prerm @@ -0,0 +1,16 @@ +#!/bin/sh + +set -e + +VERSION=${DPKG_MAINTSCRIPT_PACKAGE##*-} + +#DEBHELPER# + +. /usr/share/postgresql-common/maintscripts-functions + +stop_version $VERSION + +if [ "$1" = remove ]; then + remove_version $VERSION +fi + diff --git a/debian/postgresql-18.templates b/debian/postgresql-18.templates new file mode 100644 index 00000000000..5e75b075b90 --- /dev/null +++ b/debian/postgresql-18.templates @@ -0,0 +1,7 @@ +Template: postgresql-18/postrm_purge_data +Type: boolean +Default: true +_Description: Remove PostgreSQL directories when package is purged? + Removing the PostgreSQL server package will leave existing database clusters + intact, i.e. their configuration, data, and log directories will not be + removed. On purging the package, the directories can optionally be removed. diff --git a/debian/postgresql-client-18.install b/debian/postgresql-client-18.install new file mode 100644 index 00000000000..aef5ebf31bc --- /dev/null +++ b/debian/postgresql-client-18.install @@ -0,0 +1,48 @@ +usr/lib/postgresql/*/bin/clusterdb +usr/lib/postgresql/*/bin/createdb +usr/lib/postgresql/*/bin/createuser +usr/lib/postgresql/*/bin/dropdb +usr/lib/postgresql/*/bin/dropuser +usr/lib/postgresql/*/bin/pg_amcheck +usr/lib/postgresql/*/bin/pg_basebackup +usr/lib/postgresql/*/bin/pg_combinebackup +usr/lib/postgresql/*/bin/pg_config +usr/lib/postgresql/*/bin/pg_dump +usr/lib/postgresql/*/bin/pg_dumpall +usr/lib/postgresql/*/bin/pg_isready +usr/lib/postgresql/*/bin/pg_receivewal +usr/lib/postgresql/*/bin/pg_recvlogical +usr/lib/postgresql/*/bin/pg_restore +usr/lib/postgresql/*/bin/pg_verifybackup +usr/lib/postgresql/*/bin/psql +usr/lib/postgresql/*/bin/reindexdb +usr/lib/postgresql/*/bin/vacuumdb +usr/lib/postgresql/*/lib/pgxs/* +usr/share/locale/*/LC_MESSAGES/pg_amcheck-*.mo +usr/share/locale/*/LC_MESSAGES/pg_basebackup-*.mo +usr/share/locale/*/LC_MESSAGES/pg_combinebackup-*.mo +usr/share/locale/*/LC_MESSAGES/pg_config-*.mo +usr/share/locale/*/LC_MESSAGES/pg_dump-*.mo +usr/share/locale/*/LC_MESSAGES/pg_verifybackup-*.mo +usr/share/locale/*/LC_MESSAGES/pgscripts-*.mo +usr/share/locale/*/LC_MESSAGES/psql-*.mo +usr/share/postgresql/*/man/man1/clusterdb.1* +usr/share/postgresql/*/man/man1/createdb.1* +usr/share/postgresql/*/man/man1/createuser.1* +usr/share/postgresql/*/man/man1/dropdb.1* +usr/share/postgresql/*/man/man1/dropuser.1* +usr/share/postgresql/*/man/man1/pg_amcheck.1* +usr/share/postgresql/*/man/man1/pg_basebackup.1* +usr/share/postgresql/*/man/man1/pg_combinebackup.1* +usr/share/postgresql/*/man/man1/pg_dump.1* +usr/share/postgresql/*/man/man1/pg_dumpall.1* +usr/share/postgresql/*/man/man1/pg_isready.1* +usr/share/postgresql/*/man/man1/pg_receivewal.1* +usr/share/postgresql/*/man/man1/pg_recvlogical.1* +usr/share/postgresql/*/man/man1/pg_restore.1* +usr/share/postgresql/*/man/man1/pg_verifybackup.1* +usr/share/postgresql/*/man/man1/psql.1* +usr/share/postgresql/*/man/man1/reindexdb.1* +usr/share/postgresql/*/man/man1/vacuumdb.1* +usr/share/postgresql/*/man/man7/ +usr/share/postgresql/*/psqlrc.sample diff --git a/debian/postgresql-client-18.lintian-overrides b/debian/postgresql-client-18.lintian-overrides new file mode 100644 index 00000000000..a3cef5bfc6c --- /dev/null +++ b/debian/postgresql-client-18.lintian-overrides @@ -0,0 +1,3 @@ +# We ship binaries and libs in subdirs of /usr/lib/postgresql +executable-in-usr-lib +repeated-path-segment lib * diff --git a/debian/postgresql-client-18.postinst b/debian/postgresql-client-18.postinst new file mode 100644 index 00000000000..a5cf251f4f4 --- /dev/null +++ b/debian/postgresql-client-18.postinst @@ -0,0 +1,13 @@ +#!/bin/sh + +set -e + +VERSION=${DPKG_MAINTSCRIPT_PACKAGE##*-} + +. /usr/share/postgresql-common/maintscripts-functions + +if [ "$1" = configure ]; then + configure_client_version $VERSION "$2" +fi + +#DEBHELPER# diff --git a/debian/postgresql-client-18.prerm b/debian/postgresql-client-18.prerm new file mode 100644 index 00000000000..14c21887943 --- /dev/null +++ b/debian/postgresql-client-18.prerm @@ -0,0 +1,12 @@ +#!/bin/sh + +set -e + +VERSION=${DPKG_MAINTSCRIPT_PACKAGE##*-} + +#DEBHELPER# + +if [ "$1" = remove ]; then + . /usr/share/postgresql-common/maintscripts-functions + remove_client_version $VERSION +fi diff --git a/debian/postgresql-doc-18.doc-base b/debian/postgresql-doc-18.doc-base new file mode 100644 index 00000000000..c0e42be83f6 --- /dev/null +++ b/debian/postgresql-doc-18.doc-base @@ -0,0 +1,18 @@ +Document: postgresql-18 +Title: PostgreSQL 18 Documentation +Author: The PostgreSQL Global Development Group +Abstract: The documentation for the PostgreSQL database management system, + version 18. PostgreSQL is a powerful, open source object-relational database + system. It is fully ACID compliant, has full support for foreign keys, joins, + views, triggers, and stored procedures (in multiple languages). It includes + most SQL:2008 data types, including INTEGER, NUMERIC, BOOLEAN, CHAR, VARCHAR, + DATE, INTERVAL, and TIMESTAMP. It also supports storage of binary large + objects, including pictures, sounds, or video. It has native programming + interfaces for C/C++, Java, .Net, Perl, Python, Ruby, Tcl, ODBC, among others, + and exceptional documentation. +Section: Data Management + +Format: HTML +Index: /usr/share/doc/postgresql-doc-18/html/index.html +Files: /usr/share/doc/postgresql-doc-18/html/* + diff --git a/debian/postgresql-doc-18.install b/debian/postgresql-doc-18.install new file mode 100644 index 00000000000..a41ef190d3f --- /dev/null +++ b/debian/postgresql-doc-18.install @@ -0,0 +1,2 @@ +usr/share/doc/postgresql-doc-* +usr/share/postgresql/*/man/man3/ diff --git a/debian/postgresql-doc-18.postinst b/debian/postgresql-doc-18.postinst new file mode 100644 index 00000000000..d647cf43e8d --- /dev/null +++ b/debian/postgresql-doc-18.postinst @@ -0,0 +1,30 @@ +#!/bin/sh + +set -e + +VERSION=${DPKG_MAINTSCRIPT_PACKAGE##*-} + +# arguments: version master package [package] +__link_manpages() { + MANS=$(unset GREP_OPTIONS; dpkg -L $3 $4 $5 2>/dev/null | grep -E '/man/.*\.[1-9](\.gz)?$' | grep -v "$2") || true + [ -n "$MANS" ] || return 0 + + SLAVES=$(for i in $MANS; do TARGET=$(echo $i | sed "s/postgresql\/$1\///"); echo -n " --slave $TARGET $(basename $i) $i"; done) + + mkdir -p /usr/share/man/man3 + section=$(echo "$2" | sed -e 's/.*\.\(.*\)\..*/man\1/') + update-alternatives --install /usr/share/man/$section/$2 \ + $2 /usr/share/postgresql/$1/man/$section/$2 \ + ${1}0 $SLAVES +} + +if [ "$1" = configure ]; then + if [ -f /usr/share/postgresql-common/maintscripts-functions ]; then + . /usr/share/postgresql-common/maintscripts-functions + configure_doc_version $VERSION "$2" + else + __link_manpages $VERSION SPI_connect.3.gz "postgresql-doc-$VERSION" + fi +fi + +#DEBHELPER# diff --git a/debian/postgresql-doc-18.prerm b/debian/postgresql-doc-18.prerm new file mode 100644 index 00000000000..cae8f50aba6 --- /dev/null +++ b/debian/postgresql-doc-18.prerm @@ -0,0 +1,18 @@ +#!/bin/sh + +set -e + +VERSION=${DPKG_MAINTSCRIPT_PACKAGE##*-} + +if [ "$1" = remove ]; then + if [ -f /usr/share/postgresql-common/maintscripts-functions ]; then + . /usr/share/postgresql-common/maintscripts-functions + remove_doc_version $VERSION + else + update-alternatives --remove SPI_connect.3.gz \ + /usr/share/postgresql/$VERSION/man/man3/SPI_connect.3.gz + rmdir --ignore-fail-on-non-empty -p /usr/share/man/man1 /usr/share/man/man3 /usr/share/man/man7 + fi +fi + +#DEBHELPER# diff --git a/debian/postgresql-plperl-18.install b/debian/postgresql-plperl-18.install new file mode 100644 index 00000000000..650c9756edc --- /dev/null +++ b/debian/postgresql-plperl-18.install @@ -0,0 +1,3 @@ +usr/lib/postgresql/*/lib/*plperl*.so +usr/share/locale/*/*/plperl-*.mo +usr/share/postgresql/*/extension/*plperl* diff --git a/debian/postgresql-plperl-18.lintian-overrides b/debian/postgresql-plperl-18.lintian-overrides new file mode 120000 index 00000000000..d331563ffeb --- /dev/null +++ b/debian/postgresql-plperl-18.lintian-overrides @@ -0,0 +1 @@ +postgresql-18.lintian-overrides \ No newline at end of file diff --git a/debian/postgresql-plpython3-18.install b/debian/postgresql-plpython3-18.install new file mode 100644 index 00000000000..287a0c86376 --- /dev/null +++ b/debian/postgresql-plpython3-18.install @@ -0,0 +1,3 @@ +usr/lib/postgresql/*/lib/*plpython3*.so +usr/share/locale/*/*/plpython-*.mo +usr/share/postgresql/*/extension/*plpython3* diff --git a/debian/postgresql-plpython3-18.lintian-overrides b/debian/postgresql-plpython3-18.lintian-overrides new file mode 120000 index 00000000000..d331563ffeb --- /dev/null +++ b/debian/postgresql-plpython3-18.lintian-overrides @@ -0,0 +1 @@ +postgresql-18.lintian-overrides \ No newline at end of file diff --git a/debian/postgresql-pltcl-18.install b/debian/postgresql-pltcl-18.install new file mode 100644 index 00000000000..f56fc0fb4d4 --- /dev/null +++ b/debian/postgresql-pltcl-18.install @@ -0,0 +1,3 @@ +usr/lib/postgresql/*/lib/pltcl.so +usr/share/locale/*/*/pltcl-*.mo +usr/share/postgresql/*/extension/pltcl* diff --git a/debian/postgresql-pltcl-18.lintian-overrides b/debian/postgresql-pltcl-18.lintian-overrides new file mode 120000 index 00000000000..d331563ffeb --- /dev/null +++ b/debian/postgresql-pltcl-18.lintian-overrides @@ -0,0 +1 @@ +postgresql-18.lintian-overrides \ No newline at end of file diff --git a/debian/postgresql-server-dev-18.install b/debian/postgresql-server-dev-18.install new file mode 100644 index 00000000000..0707deee405 --- /dev/null +++ b/debian/postgresql-server-dev-18.install @@ -0,0 +1,5 @@ +usr/include/postgresql/*/server +usr/lib/postgresql/*/bin/pg_bsd_indent +usr/lib/postgresql/*/bin/pgindent +usr/lib/postgresql/*/lib/libpg*.a +usr/share/postgresql/*/typedefs.list diff --git a/debian/postgresql-server-dev-18.lintian-overrides b/debian/postgresql-server-dev-18.lintian-overrides new file mode 120000 index 00000000000..d331563ffeb --- /dev/null +++ b/debian/postgresql-server-dev-18.lintian-overrides @@ -0,0 +1 @@ +postgresql-18.lintian-overrides \ No newline at end of file diff --git a/debian/rules b/debian/rules new file mode 100755 index 00000000000..eefe16b1ea5 --- /dev/null +++ b/debian/rules @@ -0,0 +1,5 @@ +#!/usr/bin/make -f + +MAJOR_VER := 18 + +include /usr/share/postgresql-common/server/postgresql.mk diff --git a/debian/source/format b/debian/source/format new file mode 100644 index 00000000000..163aaf8d82b --- /dev/null +++ b/debian/source/format @@ -0,0 +1 @@ +3.0 (quilt) diff --git a/debian/source/lintian-overrides b/debian/source/lintian-overrides new file mode 100644 index 00000000000..a244735a805 --- /dev/null +++ b/debian/source/lintian-overrides @@ -0,0 +1,4 @@ +# pregenerated docs contain some tables rendered on a single line +source: source-is-missing [doc/src/sgml/html/*] +# same for these, plus including some regression output files +source: very-long-line-length-in-source-file * diff --git a/debian/tests/Makefile.regress b/debian/tests/Makefile.regress new file mode 100644 index 00000000000..f01a80a52cf --- /dev/null +++ b/debian/tests/Makefile.regress @@ -0,0 +1,5 @@ +MODULE_big = regress +OBJS = regress.o +PG_CONFIG = pg_config +PGXS = $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) diff --git a/debian/tests/control b/debian/tests/control new file mode 100644 index 00000000000..c05e0299092 --- /dev/null +++ b/debian/tests/control @@ -0,0 +1,21 @@ +Tests: run-testsuite +Depends: + build-essential, + debhelper, + fakeroot, + hunspell-en-us, + iproute2, + locales-all, + logrotate, + netcat-openbsd, + perl, + procps, + @, +Restrictions: needs-root + +Tests: installcheck +Depends: + build-essential, + tzdata-legacy | tzdata (<< 2023c-8), + @, +Restrictions: allow-stderr diff --git a/debian/tests/installcheck b/debian/tests/installcheck new file mode 100755 index 00000000000..fc390ea14a9 --- /dev/null +++ b/debian/tests/installcheck @@ -0,0 +1,41 @@ +#!/bin/sh + +set -eux + +SOURCE=$(dpkg-parsechangelog -SSource) +MAJOR=${SOURCE#*-} +top_srcdir=$PWD + +cd src/test/regress + +# compile regress.so +make -f $top_srcdir/debian/tests/Makefile.regress PG_CONFIG=/usr/lib/postgresql/$MAJOR/bin/pg_config with_llvm=no + +# tell regression files that regress.so is not installed +sed -i -e "s;set regresslib :libdir;set regresslib '$PWD';" sql/* expected/* + +# when root, execute testsuite as user postgres since it insists on wiping the tablespace directory +if [ $(id -u) = 0 ]; then + SU="su postgres" +else + SU="sh" +fi + +$SU < Date: Tue, 17 Jun 2025 18:37:11 +0000 Subject: [PATCH 02/45] Apply debian patch series --- HISTORY | 3 +++ configure.ac | 5 +---- src/Makefile.global.in | 6 +++--- src/backend/utils/activity/pgstat_database.c | 14 +++++++++++++- src/bin/psql/settings.h | 4 ++-- src/common/Makefile | 12 ++++++------ src/fe_utils/Makefile | 6 +++--- src/include/fe_utils/print.h | 2 +- src/include/pg_config_manual.h | 2 +- src/include/port/pg_iovec.h | 5 +++++ src/interfaces/libpq/pg_service.conf.sample | 4 ++-- src/port/Makefile | 10 +++++----- src/tutorial/README | 3 +-- 13 files changed, 46 insertions(+), 30 deletions(-) diff --git a/HISTORY b/HISTORY index b87be55abf4..f12fd89e960 100644 --- a/HISTORY +++ b/HISTORY @@ -3,3 +3,6 @@ https://www.postgresql.org/docs/current/release.html Distribution file sets include release notes for their version and preceding versions. Visit the file doc/src/sgml/html/release.html in an HTML browser. + +On Debian systems, the release notes are contained in the postgresql-doc-* +packages, located in /usr/share/doc/postgresql-doc-*/html/release.html. diff --git a/configure.ac b/configure.ac index 2335947522d..6266944a32e 100644 --- a/configure.ac +++ b/configure.ac @@ -19,14 +19,11 @@ m4_pattern_forbid(^PGAC_)dnl to catch undefined macros AC_INIT([PostgreSQL], [18.3], [pgsql-bugs@lists.postgresql.org], [], [https://www.postgresql.org/]) -m4_if(m4_defn([m4_PACKAGE_VERSION]), [2.69], [], [m4_fatal([Autoconf version 2.69 is required. -Untested combinations of 'autoconf' and PostgreSQL versions are not -recommended. You can remove the check from 'configure.ac' but it is then -your responsibility whether the result works or not.])]) AC_COPYRIGHT([Copyright (c) 1996-2025, PostgreSQL Global Development Group]) AC_CONFIG_SRCDIR([src/backend/access/common/heaptuple.c]) AC_CONFIG_AUX_DIR(config) AC_PREFIX_DEFAULT(/usr/local/pgsql) +[ac_configure_args=$(echo "$ac_configure_args" | sed -e "s/ -f\(debug\|file\)-prefix-map=[^' ]*//g")] AC_DEFINE_UNQUOTED(CONFIGURE_ARGS, ["$ac_configure_args"], [Saved arguments from configure]) [PG_MAJORVERSION=`expr "$PACKAGE_VERSION" : '\([0-9][0-9]*\)'`] diff --git a/src/Makefile.global.in b/src/Makefile.global.in index 8b1b357beaa..2409005e532 100644 --- a/src/Makefile.global.in +++ b/src/Makefile.global.in @@ -129,7 +129,7 @@ libdir := @libdir@ pkglibdir = $(libdir) ifeq "$(findstring pgsql, $(pkglibdir))" "" ifeq "$(findstring postgres, $(pkglibdir))" "" -override pkglibdir := $(pkglibdir)/postgresql +override pkglibdir := /usr/lib/postgresql/@PG_MAJORVERSION@/lib endif endif @@ -172,7 +172,7 @@ endif # PGXS # These derived path variables aren't separately configurable. -includedir_server = $(pkgincludedir)/server +includedir_server = $(pkgincludedir)/@PG_MAJORVERSION@/server includedir_internal = $(pkgincludedir)/internal pgxsdir = $(pkglibdir)/pgxs bitcodedir = $(pkglibdir)/bitcode @@ -625,7 +625,7 @@ libpq_pgport = $(libpq) # done if they don't, since they will have satisfied all their references # from these libraries.) ifdef PGXS -libpq_pgport_shlib = -L$(libdir) -lpgcommon_shlib -lpgport_shlib $(libpq) +libpq_pgport_shlib = -L$(pkglibdir) -lpgcommon_shlib -lpgport_shlib $(libpq) else libpq_pgport_shlib = -L$(top_builddir)/src/common -lpgcommon_shlib -L$(top_builddir)/src/port -lpgport_shlib $(libpq) endif diff --git a/src/backend/utils/activity/pgstat_database.c b/src/backend/utils/activity/pgstat_database.c index 9f1a8633b4b..7ef3b98754f 100644 --- a/src/backend/utils/activity/pgstat_database.c +++ b/src/backend/utils/activity/pgstat_database.c @@ -81,12 +81,22 @@ void pgstat_report_recovery_conflict(int reason) { PgStat_StatDBEntry *dbentry; + PgStat_EntryRef *entry_ref; + PgStatShared_Database *sharedent; Assert(IsUnderPostmaster); if (!pgstat_track_counts) return; - dbentry = pgstat_prep_database_pending(MyDatabaseId); + /* + * Update the shared stats directly - recovery conflicts should never be + * common enough for that to be a problem. + */ + entry_ref = + pgstat_get_entry_ref_locked(PGSTAT_KIND_DATABASE, MyDatabaseId, InvalidOid, false); + + sharedent = (PgStatShared_Database *) entry_ref->shared_stats; + dbentry = &sharedent->stats; switch (reason) { @@ -116,6 +126,8 @@ pgstat_report_recovery_conflict(int reason) dbentry->conflict_startup_deadlock++; break; } + + pgstat_unlock_entry(entry_ref); } /* diff --git a/src/bin/psql/settings.h b/src/bin/psql/settings.h index fd82303f776..4df4059ff7e 100644 --- a/src/bin/psql/settings.h +++ b/src/bin/psql/settings.h @@ -19,8 +19,8 @@ #define DEFAULT_EDITOR "notepad.exe" /* no DEFAULT_EDITOR_LINENUMBER_ARG for Notepad */ #else -#define DEFAULT_EDITOR "vi" -#define DEFAULT_EDITOR_LINENUMBER_ARG "+" +#define DEFAULT_EDITOR "sensible-editor" +/*#define DEFAULT_EDITOR_LINENUMBER_ARG "+"*/ #endif #define DEFAULT_PROMPT1 "%/%R%x%# " diff --git a/src/common/Makefile b/src/common/Makefile index 2c720caa509..4dac609bdca 100644 --- a/src/common/Makefile +++ b/src/common/Makefile @@ -33,7 +33,7 @@ STD_CPPFLAGS := $(filter-out -I$(top_srcdir)/src/include -I$(top_builddir)/src/i STD_LDFLAGS := $(filter-out -L$(top_builddir)/src/common -L$(top_builddir)/src/port,$(LDFLAGS)) override CPPFLAGS += -DVAL_CC="\"$(CC)\"" override CPPFLAGS += -DVAL_CPPFLAGS="\"$(STD_CPPFLAGS)\"" -override CPPFLAGS += -DVAL_CFLAGS="\"$(CFLAGS)\"" +override CPPFLAGS += -DVAL_CFLAGS="\"$(filter-out -fdebug-prefix-map=% -ffile-prefix-map=%,$(CFLAGS))\"" override CPPFLAGS += -DVAL_CFLAGS_SL="\"$(CFLAGS_SL)\"" override CPPFLAGS += -DVAL_LDFLAGS="\"$(STD_LDFLAGS)\"" override CPPFLAGS += -DVAL_LDFLAGS_EX="\"$(LDFLAGS_EX)\"" @@ -131,15 +131,15 @@ all: libpgcommon.a libpgcommon_shlib.a libpgcommon_srv.a libpgcommon_excluded_sh # libpgcommon is needed by some contrib install: all installdirs - $(INSTALL_STLIB) libpgcommon.a '$(DESTDIR)$(libdir)/libpgcommon.a' - $(INSTALL_STLIB) libpgcommon_shlib.a '$(DESTDIR)$(libdir)/libpgcommon_shlib.a' + $(INSTALL_STLIB) libpgcommon.a '$(DESTDIR)$(pkglibdir)/libpgcommon.a' + $(INSTALL_STLIB) libpgcommon_shlib.a '$(DESTDIR)$(pkglibdir)/libpgcommon_shlib.a' installdirs: - $(MKDIR_P) '$(DESTDIR)$(libdir)' + $(MKDIR_P) '$(DESTDIR)$(pkglibdir)' uninstall: - rm -f '$(DESTDIR)$(libdir)/libpgcommon.a' - rm -f '$(DESTDIR)$(libdir)/libpgcommon_shlib.a' + rm -f '$(DESTDIR)$(pkglibdir)/libpgcommon.a' + rm -f '$(DESTDIR)$(pkglibdir)/libpgcommon_shlib.a' libpgcommon.a: $(OBJS_FRONTEND) rm -f $@ diff --git a/src/fe_utils/Makefile b/src/fe_utils/Makefile index 28196ce0f6a..48c4ce4039f 100644 --- a/src/fe_utils/Makefile +++ b/src/fe_utils/Makefile @@ -54,13 +54,13 @@ psqlscan.c: FLEX_NO_BACKUP=yes # libpgfeutils could be useful to contrib, so install it install: all installdirs - $(INSTALL_STLIB) libpgfeutils.a '$(DESTDIR)$(libdir)/libpgfeutils.a' + $(INSTALL_STLIB) libpgfeutils.a '$(DESTDIR)$(pkglibdir)/libpgfeutils.a' installdirs: - $(MKDIR_P) '$(DESTDIR)$(libdir)' + $(MKDIR_P) '$(DESTDIR)$(pkglibdir)' uninstall: - rm -f '$(DESTDIR)$(libdir)/libpgfeutils.a' + rm -f '$(DESTDIR)$(pkglibdir)/libpgfeutils.a' clean distclean: rm -f libpgfeutils.a $(OBJS) lex.backup diff --git a/src/include/fe_utils/print.h b/src/include/fe_utils/print.h index c99c2ee1a31..22ffcb3e5e1 100644 --- a/src/include/fe_utils/print.h +++ b/src/include/fe_utils/print.h @@ -20,7 +20,7 @@ /* This is not a particularly great place for this ... */ #ifndef __CYGWIN__ -#define DEFAULT_PAGER "more" +#define DEFAULT_PAGER "pager" #else #define DEFAULT_PAGER "less" #endif diff --git a/src/include/pg_config_manual.h b/src/include/pg_config_manual.h index 125d3eb5fff..b8677324731 100644 --- a/src/include/pg_config_manual.h +++ b/src/include/pg_config_manual.h @@ -190,7 +190,7 @@ * support them yet. */ #ifndef WIN32 -#define DEFAULT_PGSOCKET_DIR "/tmp" +#define DEFAULT_PGSOCKET_DIR "/var/run/postgresql" #else #define DEFAULT_PGSOCKET_DIR "" #endif diff --git a/src/include/port/pg_iovec.h b/src/include/port/pg_iovec.h index 90be3af449d..b42766b68a0 100644 --- a/src/include/port/pg_iovec.h +++ b/src/include/port/pg_iovec.h @@ -19,6 +19,11 @@ #include /* IWYU pragma: export */ #include +/* POSIX requires at least 16 as a maximum iovcnt. */ +#ifndef IOV_MAX +#define IOV_MAX 16 +#endif + #else /* Define our own POSIX-compatible iovec struct. */ diff --git a/src/interfaces/libpq/pg_service.conf.sample b/src/interfaces/libpq/pg_service.conf.sample index 5a1c083538b..7ef2ebde701 100644 --- a/src/interfaces/libpq/pg_service.conf.sample +++ b/src/interfaces/libpq/pg_service.conf.sample @@ -8,8 +8,8 @@ # to look up such parameters. A sample configuration for postgres is # included in this file. Lines beginning with '#' are comments. # -# Copy this to your sysconf directory (typically /usr/local/pgsql/etc) and -# rename it pg_service.conf. +# Copy this to /etc/postgresql-common/ (or select its location with the +# PGSYSCONFDIR environment variable) and rename it pg_service.conf. # # #[postgres] diff --git a/src/port/Makefile b/src/port/Makefile index 4274949dfa4..7b5e27759d3 100644 --- a/src/port/Makefile +++ b/src/port/Makefile @@ -71,15 +71,15 @@ all: libpgport.a libpgport_shlib.a libpgport_srv.a # libpgport is needed by some contrib install: all installdirs - $(INSTALL_STLIB) libpgport.a '$(DESTDIR)$(libdir)/libpgport.a' - $(INSTALL_STLIB) libpgport_shlib.a '$(DESTDIR)$(libdir)/libpgport_shlib.a' + $(INSTALL_STLIB) libpgport.a '$(DESTDIR)$(pkglibdir)/libpgport.a' + $(INSTALL_STLIB) libpgport_shlib.a '$(DESTDIR)$(pkglibdir)/libpgport_shlib.a' installdirs: - $(MKDIR_P) '$(DESTDIR)$(libdir)' + $(MKDIR_P) '$(DESTDIR)$(pkglibdir)' uninstall: - rm -f '$(DESTDIR)$(libdir)/libpgport.a' - rm -f '$(DESTDIR)$(libdir)/libpgport_shlib.a' + rm -f '$(DESTDIR)$(pkglibdir)/libpgport.a' + rm -f '$(DESTDIR)$(pkglibdir)/libpgport_shlib.a' libpgport.a: $(OBJS) rm -f $@ diff --git a/src/tutorial/README b/src/tutorial/README index b137cdfad34..be070c573db 100644 --- a/src/tutorial/README +++ b/src/tutorial/README @@ -6,8 +6,7 @@ tutorial This directory contains SQL tutorial scripts. To look at them, first do a % make to compile all the scripts and C files for the user-defined functions -and types. (make needs to be GNU make --- it may be named something -different on your system, often 'gmake') +and types. This requires a postgresql-server-dev-* package to be installed. Then, run psql with the -s (single-step) flag: % psql -s From d9b99b06ae2a6a16610c5b15e0aef1c77f8efb5e Mon Sep 17 00:00:00 2001 From: reshke Date: Thu, 26 Jun 2025 11:38:51 +0000 Subject: [PATCH 03/45] Lower postgresql-common and remove tzdata restriction --- debian/control | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debian/control b/debian/control index 0e0fb1f87d1..fde5f7d9c40 100644 --- a/debian/control +++ b/debian/control @@ -44,12 +44,12 @@ Build-Depends: mawk, perl (>= 5.8), pkgconf, - postgresql-common (>= 279~), + postgresql-common (>= 277~), python3-dev, systemtap-sdt-dev [linux-any], tcl-dev, tzdata , - tzdata-legacy | tzdata (<< 2023c-8) , + tzdata-legacy | tzdata , uuid-dev, xsltproc, zlib1g-dev | libz-dev, From 72d50cdc4d8b0ddbbe07efeb0d2721ff2c4d4f1c Mon Sep 17 00:00:00 2001 From: reshke Date: Tue, 17 Jun 2025 18:37:47 +0000 Subject: [PATCH 04/45] Remove stop version --- debian/postgresql-18.prerm | 2 -- 1 file changed, 2 deletions(-) diff --git a/debian/postgresql-18.prerm b/debian/postgresql-18.prerm index 828c6fbe9f5..f8d6ac10a20 100644 --- a/debian/postgresql-18.prerm +++ b/debian/postgresql-18.prerm @@ -8,8 +8,6 @@ VERSION=${DPKG_MAINTSCRIPT_PACKAGE##*-} . /usr/share/postgresql-common/maintscripts-functions -stop_version $VERSION - if [ "$1" = remove ]; then remove_version $VERSION fi From 2ae2c0662c5dd24728cc38f73814388ae503d804 Mon Sep 17 00:00:00 2001 From: Kirill Reshke Date: Tue, 27 Feb 2024 00:29:43 +0300 Subject: [PATCH 05/45] Yandex build infra commit --- Dockerfile | 54 ++++++++++++++++++++++++++++++++++++++++++++ docker/entrypoint.sh | 24 ++++++++++++++++++++ docker/tzdata.sh | 9 ++++++++ prepare-build.sh | 17 ++++++++++++++ 4 files changed, 104 insertions(+) create mode 100644 Dockerfile create mode 100755 docker/entrypoint.sh create mode 100755 docker/tzdata.sh create mode 100755 prepare-build.sh diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000000..709a8ea0218 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,54 @@ +ARG codename +FROM ubuntu:${codename:-bionic} + +ARG codename +ENV CODE_NAME=${codename:-bionic} + +ARG pgdg +ENV PGDG_VER=${pgdg:-242-2-pgdg18.04+1+yandex220} + +ENV DEBIAN_FRONTEND=noninteractive +ENV TZ=Europe/Moskow +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + +RUN sed -i 's/archive.ubuntu.com/mirror.yandex.ru/g' /etc/apt/sources.list &&\ + apt-get update && apt-get install -y --no-install-recommends \ + sudo build-essential \ + gcc lsb-release libssl-dev gnupg openssl \ + gdb git curl + +RUN echo "deb http://dist.yandex.ru/mdb-${CODE_NAME}-secure stable/all/" >> /etc/apt/sources.list +RUN echo "deb http://dist.yandex.ru/mdb-${CODE_NAME}-secure stable/\$(ARCH)/" >> /etc/apt/sources.list + +RUN curl -s 'http://keyserver.ubuntu.com/pks/lookup?op=get&search=0xafc3ce0d00e3c45a357e9e637fcd11186050cd1a' | \ + gpg --dearmour -o /etc/apt/trusted.gpg.d/yandex.gpg + +RUN apt-get update && apt-get install -y --no-install-recommends \ + sudo build-essential \ + gcc lsb-release libssl-dev gnupg openssl \ + gdb git \ + libpam0g-dev \ + debhelper debootstrap devscripts make equivs debhelper-compat \ + libz-dev flex libicu-dev libio-pty-perl libipc-run-perl libkrb5-dev \ + libldap2-dev liblz4-dev liblz4-tool zstd libperl-dev libreadline-dev libselinux1-dev llvm-dev \ + libsystemd-dev libxml2-dev libxml2-utils libxslt1-dev \ + python3-dev systemtap-sdt-dev tcl-dev uuid-dev xsltproc zlib1g-dev \ + bison dh-exec docbook-xml docbook-xsl \ + clang libcurl4-openssl-dev libnuma-dev liburing-dev libzstd-dev pkgconf tzdata +RUN apt-get install -y \ + libmdblocales1 libmdblocales-dev \ + postgresql-client-common=${PGDG_VER} \ + postgresql-common=${PGDG_VER} \ + postgresql-common-dev=${PGDG_VER} + +RUN groupadd -g 999 build-user && \ + useradd -r -u 999 -g build-user build-user + +COPY . /home/build-user +RUN chown build-user:build-user /home -R && usermod -aG sudo build-user + +RUN echo 'build-user ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers + +USER build-user + +ENTRYPOINT ["/home/build-user/docker/entrypoint.sh"] diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh new file mode 100755 index 00000000000..f887448fe50 --- /dev/null +++ b/docker/entrypoint.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +set -ex + +export DEBIAN_FRONTEND=noninteractive +export TZ=Europe/Moskow +sudo bash -c "echo $TZ > /etc/timezone" + +cd /home/build-user + +sudo ./docker/tzdata.sh + +cat debian/changelog +#export DEB_BUILD_OPTIONS="nocheck" + +sudo dpkg-checkbuilddeps + +sudo mk-build-deps --build-dep --install --tool='apt-get -o Debug::pkgProblemResolver=yes --no-install-recommends --yes' debian/control + +dpkg-buildpackage -b -rfakeroot -us -uc +#dpkg-buildpackage -us -uc + +cd /home +rm -fr build-user + diff --git a/docker/tzdata.sh b/docker/tzdata.sh new file mode 100755 index 00000000000..004e6de4101 --- /dev/null +++ b/docker/tzdata.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +export DEBIAN_FRONTEND=noninteractive +#install tzdata package +apt-get install -y tzdata +# set your timezone +ln -fs /usr/share/zoneinfo/$TZ /etc/localtime +dpkg-reconfigure --frontend noninteractive tzdata + diff --git a/prepare-build.sh b/prepare-build.sh new file mode 100755 index 00000000000..5d41acff2aa --- /dev/null +++ b/prepare-build.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +export PACKAGE_NAME=postgresql-18 +export BUILD_USER=mdb-cc +export VERSION=$(grep 'PACKAGE_VERSION=' configure | cut -d= -f2 | sed s/\'//g)-201-yandex.$(git rev-list HEAD --count).$(git rev-parse --short HEAD) +export LC_ALL=C + +cat > debian/changelog< $(date +'%a, %d %b %Y %H:%M:%S %z') +EOH + +echo "VERSION=$VERSION" > version.properties + From 9a1f21767311b5a4974ef652db94910ce134eb53 Mon Sep 17 00:00:00 2001 From: Kirill Reshke Date: Tue, 26 Dec 2023 14:43:49 +0300 Subject: [PATCH 06/45] Untrust all contrib --- contrib/bool_plperl/bool_plperl.control | 2 +- contrib/btree_gin/btree_gin.control | 2 +- contrib/btree_gist/btree_gist.control | 2 +- contrib/citext/citext.control | 2 +- contrib/cube/cube.control | 2 +- contrib/dict_int/dict_int.control | 2 +- contrib/fuzzystrmatch/fuzzystrmatch.control | 2 +- contrib/hstore/hstore.control | 2 +- contrib/intarray/intarray.control | 2 +- contrib/isn/isn.control | 2 +- contrib/jsonb_plperl/jsonb_plperl.control | 2 +- contrib/lo/lo.control | 2 +- contrib/ltree/ltree.control | 2 +- contrib/pg_trgm/pg_trgm.control | 2 +- contrib/pgcrypto/pgcrypto.control | 2 +- contrib/seg/seg.control | 2 +- contrib/tablefunc/tablefunc.control | 2 +- contrib/tcn/tcn.control | 2 +- contrib/tsm_system_rows/tsm_system_rows.control | 2 +- contrib/tsm_system_time/tsm_system_time.control | 2 +- contrib/unaccent/unaccent.control | 2 +- contrib/uuid-ossp/uuid-ossp.control | 2 +- 22 files changed, 22 insertions(+), 22 deletions(-) diff --git a/contrib/bool_plperl/bool_plperl.control b/contrib/bool_plperl/bool_plperl.control index af3e6b1966f..65f9755adad 100644 --- a/contrib/bool_plperl/bool_plperl.control +++ b/contrib/bool_plperl/bool_plperl.control @@ -3,5 +3,5 @@ comment = 'transform between bool and plperl' default_version = '1.0' module_pathname = '$libdir/bool_plperl' relocatable = true -trusted = true +trusted = false requires = 'plperl' diff --git a/contrib/btree_gin/btree_gin.control b/contrib/btree_gin/btree_gin.control index 67d0c997d8d..5fc20d2cd65 100644 --- a/contrib/btree_gin/btree_gin.control +++ b/contrib/btree_gin/btree_gin.control @@ -3,4 +3,4 @@ comment = 'support for indexing common datatypes in GIN' default_version = '1.3' module_pathname = '$libdir/btree_gin' relocatable = true -trusted = true +trusted = false diff --git a/contrib/btree_gist/btree_gist.control b/contrib/btree_gist/btree_gist.control index abf66538f32..3f73ecf65f4 100644 --- a/contrib/btree_gist/btree_gist.control +++ b/contrib/btree_gist/btree_gist.control @@ -3,4 +3,4 @@ comment = 'support for indexing common datatypes in GiST' default_version = '1.8' module_pathname = '$libdir/btree_gist' relocatable = true -trusted = true +trusted = false diff --git a/contrib/citext/citext.control b/contrib/citext/citext.control index 2b0f3fa8407..f26c8b451b5 100644 --- a/contrib/citext/citext.control +++ b/contrib/citext/citext.control @@ -3,4 +3,4 @@ comment = 'data type for case-insensitive character strings' default_version = '1.8' module_pathname = '$libdir/citext' relocatable = true -trusted = true +trusted = false diff --git a/contrib/cube/cube.control b/contrib/cube/cube.control index 50427ec1170..cf21f9e5fd9 100644 --- a/contrib/cube/cube.control +++ b/contrib/cube/cube.control @@ -3,4 +3,4 @@ comment = 'data type for multidimensional cubes' default_version = '1.5' module_pathname = '$libdir/cube' relocatable = true -trusted = true +trusted = false diff --git a/contrib/dict_int/dict_int.control b/contrib/dict_int/dict_int.control index ec04ccea91a..1ca11400d41 100644 --- a/contrib/dict_int/dict_int.control +++ b/contrib/dict_int/dict_int.control @@ -3,4 +3,4 @@ comment = 'text search dictionary template for integers' default_version = '1.0' module_pathname = '$libdir/dict_int' relocatable = true -trusted = true +trusted = false diff --git a/contrib/fuzzystrmatch/fuzzystrmatch.control b/contrib/fuzzystrmatch/fuzzystrmatch.control index 8b6e9fd9935..e0ffadcbddb 100644 --- a/contrib/fuzzystrmatch/fuzzystrmatch.control +++ b/contrib/fuzzystrmatch/fuzzystrmatch.control @@ -3,4 +3,4 @@ comment = 'determine similarities and distance between strings' default_version = '1.2' module_pathname = '$libdir/fuzzystrmatch' relocatable = true -trusted = true +trusted = false diff --git a/contrib/hstore/hstore.control b/contrib/hstore/hstore.control index 89e3c746c46..248223b9558 100644 --- a/contrib/hstore/hstore.control +++ b/contrib/hstore/hstore.control @@ -3,4 +3,4 @@ comment = 'data type for storing sets of (key, value) pairs' default_version = '1.8' module_pathname = '$libdir/hstore' relocatable = true -trusted = true +trusted = false diff --git a/contrib/intarray/intarray.control b/contrib/intarray/intarray.control index c3ff753e2cf..e01110f2681 100644 --- a/contrib/intarray/intarray.control +++ b/contrib/intarray/intarray.control @@ -3,4 +3,4 @@ comment = 'functions, operators, and index support for 1-D arrays of integers' default_version = '1.5' module_pathname = '$libdir/_int' relocatable = true -trusted = true +trusted = false diff --git a/contrib/isn/isn.control b/contrib/isn/isn.control index e7daea52b84..ff15cc5f331 100644 --- a/contrib/isn/isn.control +++ b/contrib/isn/isn.control @@ -3,4 +3,4 @@ comment = 'data types for international product numbering standards' default_version = '1.3' module_pathname = '$libdir/isn' relocatable = true -trusted = true +trusted = false diff --git a/contrib/jsonb_plperl/jsonb_plperl.control b/contrib/jsonb_plperl/jsonb_plperl.control index 4acee93a2fc..b4adfad17fb 100644 --- a/contrib/jsonb_plperl/jsonb_plperl.control +++ b/contrib/jsonb_plperl/jsonb_plperl.control @@ -3,5 +3,5 @@ comment = 'transform between jsonb and plperl' default_version = '1.0' module_pathname = '$libdir/jsonb_plperl' relocatable = true -trusted = true +trusted = false requires = 'plperl' diff --git a/contrib/lo/lo.control b/contrib/lo/lo.control index 3acb3665a4e..1e992e36957 100644 --- a/contrib/lo/lo.control +++ b/contrib/lo/lo.control @@ -3,4 +3,4 @@ comment = 'Large Object maintenance' default_version = '1.2' module_pathname = '$libdir/lo' relocatable = true -trusted = true +trusted = false diff --git a/contrib/ltree/ltree.control b/contrib/ltree/ltree.control index c2cbeda96c7..97f697fad62 100644 --- a/contrib/ltree/ltree.control +++ b/contrib/ltree/ltree.control @@ -3,4 +3,4 @@ comment = 'data type for hierarchical tree-like structures' default_version = '1.3' module_pathname = '$libdir/ltree' relocatable = true -trusted = true +trusted = false diff --git a/contrib/pg_trgm/pg_trgm.control b/contrib/pg_trgm/pg_trgm.control index 1d6a9ddf259..ba1db412f3a 100644 --- a/contrib/pg_trgm/pg_trgm.control +++ b/contrib/pg_trgm/pg_trgm.control @@ -3,4 +3,4 @@ comment = 'text similarity measurement and index searching based on trigrams' default_version = '1.6' module_pathname = '$libdir/pg_trgm' relocatable = true -trusted = true +trusted = false diff --git a/contrib/pgcrypto/pgcrypto.control b/contrib/pgcrypto/pgcrypto.control index fcdd0b46f5b..31266c6085f 100644 --- a/contrib/pgcrypto/pgcrypto.control +++ b/contrib/pgcrypto/pgcrypto.control @@ -3,4 +3,4 @@ comment = 'cryptographic functions' default_version = '1.4' module_pathname = '$libdir/pgcrypto' relocatable = true -trusted = true +trusted = false diff --git a/contrib/seg/seg.control b/contrib/seg/seg.control index e2c6a4750fc..0fa4236c079 100644 --- a/contrib/seg/seg.control +++ b/contrib/seg/seg.control @@ -3,4 +3,4 @@ comment = 'data type for representing line segments or floating-point intervals' default_version = '1.4' module_pathname = '$libdir/seg' relocatable = true -trusted = true +trusted = false diff --git a/contrib/tablefunc/tablefunc.control b/contrib/tablefunc/tablefunc.control index 7b25d161702..64d039977de 100644 --- a/contrib/tablefunc/tablefunc.control +++ b/contrib/tablefunc/tablefunc.control @@ -3,4 +3,4 @@ comment = 'functions that manipulate whole tables, including crosstab' default_version = '1.0' module_pathname = '$libdir/tablefunc' relocatable = true -trusted = true +trusted = false diff --git a/contrib/tcn/tcn.control b/contrib/tcn/tcn.control index 6972e1102e2..d54c22c9b49 100644 --- a/contrib/tcn/tcn.control +++ b/contrib/tcn/tcn.control @@ -3,4 +3,4 @@ comment = 'Triggered change notifications' default_version = '1.0' module_pathname = '$libdir/tcn' relocatable = true -trusted = true +trusted = false diff --git a/contrib/tsm_system_rows/tsm_system_rows.control b/contrib/tsm_system_rows/tsm_system_rows.control index b495fb126c0..ccc72d89058 100644 --- a/contrib/tsm_system_rows/tsm_system_rows.control +++ b/contrib/tsm_system_rows/tsm_system_rows.control @@ -3,4 +3,4 @@ comment = 'TABLESAMPLE method which accepts number of rows as a limit' default_version = '1.0' module_pathname = '$libdir/tsm_system_rows' relocatable = true -trusted = true +trusted = false diff --git a/contrib/tsm_system_time/tsm_system_time.control b/contrib/tsm_system_time/tsm_system_time.control index b1b9789debc..e0681787e33 100644 --- a/contrib/tsm_system_time/tsm_system_time.control +++ b/contrib/tsm_system_time/tsm_system_time.control @@ -3,4 +3,4 @@ comment = 'TABLESAMPLE method which accepts time in milliseconds as a limit' default_version = '1.0' module_pathname = '$libdir/tsm_system_time' relocatable = true -trusted = true +trusted = false diff --git a/contrib/unaccent/unaccent.control b/contrib/unaccent/unaccent.control index 649cf68a6e7..9cb1a520806 100644 --- a/contrib/unaccent/unaccent.control +++ b/contrib/unaccent/unaccent.control @@ -3,4 +3,4 @@ comment = 'text search dictionary that removes accents' default_version = '1.1' module_pathname = '$libdir/unaccent' relocatable = true -trusted = true +trusted = false diff --git a/contrib/uuid-ossp/uuid-ossp.control b/contrib/uuid-ossp/uuid-ossp.control index 142a99e4a89..8c2bd525b3d 100644 --- a/contrib/uuid-ossp/uuid-ossp.control +++ b/contrib/uuid-ossp/uuid-ossp.control @@ -3,4 +3,4 @@ comment = 'generate universally unique identifiers (UUIDs)' default_version = '1.1' module_pathname = '$libdir/uuid-ossp' relocatable = true -trusted = true +trusted = false From 74fd8a753c09f4d1ea9fe89438bf56e31b580bc4 Mon Sep 17 00:00:00 2001 From: Andrey Borodin Date: Tue, 27 Oct 2020 14:21:21 +0500 Subject: [PATCH 07/45] Disallow cancelation of syncronous commit V1 Currently we allow to cancel awaiting of syncronous commit. Some drivers cancel query after timeout. If application will retry idempotent query, it will get confirmation of written data. This can lead to split-brain in HA scenarios. To prevent it this we add synchronous_commit_cancelation setting disalowing cancelation of syncronous replication wait Version for PostgreSQL 16 --- src/backend/access/transam/xact.c | 1 + src/backend/replication/syncrep.c | 19 +++++++++++++------ src/backend/utils/misc/guc_tables.c | 9 +++++++++ src/backend/utils/misc/postgresql.conf.sample | 5 +++++ src/include/access/xact.h | 2 ++ 5 files changed, 30 insertions(+), 6 deletions(-) diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c index b885513f765..3e0f6d2a305 100644 --- a/src/backend/access/transam/xact.c +++ b/src/backend/access/transam/xact.c @@ -85,6 +85,7 @@ bool DefaultXactDeferrable = false; bool XactDeferrable; int synchronous_commit = SYNCHRONOUS_COMMIT_ON; +bool synchronous_commit_cancelation = false; /* * CheckXidAlive is a xid value pointing to a possibly ongoing (sub) diff --git a/src/backend/replication/syncrep.c b/src/backend/replication/syncrep.c index 576320f952f..3b4f09c6e74 100644 --- a/src/backend/replication/syncrep.c +++ b/src/backend/replication/syncrep.c @@ -301,8 +301,8 @@ SyncRepWaitForLSN(XLogRecPtr lsn, bool commit) { ereport(WARNING, (errcode(ERRCODE_ADMIN_SHUTDOWN), - errmsg("canceling the wait for synchronous replication and terminating connection due to administrator command"), - errdetail("The transaction has already committed locally, but might not have been replicated to the standby."))); + errmsg("canceling the wait for synchronous replication and terminating connection due to administrator command"), + errdetail("The transaction has already committed locally, but might not have been replicated to the standby."))); whereToSendOutput = DestNone; SyncRepCancelWait(); break; @@ -317,11 +317,18 @@ SyncRepWaitForLSN(XLogRecPtr lsn, bool commit) if (QueryCancelPending) { QueryCancelPending = false; + if (synchronous_commit_cancelation) + { + ereport(WARNING, + (errmsg("canceling wait for synchronous replication due to user request"), + errdetail("The transaction has already committed locally, but might not have been replicated to the standby."))); + SyncRepCancelWait(); + break; + } + ereport(WARNING, - (errmsg("canceling wait for synchronous replication due to user request"), - errdetail("The transaction has already committed locally, but might not have been replicated to the standby."))); - SyncRepCancelWait(); - break; + (errmsg("canceling wait for synchronous replication due requested, but cancelation is not allowed"), + errdetail("The COMMIT record has already flushed to WAL locally and might not have been replicated to the standby. We must wait here."))); } /* diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index 6b82a23435e..5b0cdc70942 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -1235,6 +1235,15 @@ struct config_bool ConfigureNamesBool[] = NULL, NULL, NULL }, + { + {"synchronous_commit_cancelation", PGC_USERSET, WAL_SETTINGS, + gettext_noop("Allow to cancel waiting for replication of transaction commited localy."), + NULL + }, + &synchronous_commit_cancelation, + false, NULL, NULL, NULL + }, + { {"log_checkpoints", PGC_SIGHUP, LOGGING_WHAT, gettext_noop("Logs each checkpoint."), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index d91133dbd73..f1a862b9b8c 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -880,6 +880,11 @@ #include_if_exists = '...' # include file only if it exists #include = '...' # include file +#------------------------------------------------------------------------------ +# MDB +#------------------------------------------------------------------------------ + +#synchronous_commit_cancelation = off #------------------------------------------------------------------------------ # CUSTOMIZED OPTIONS diff --git a/src/include/access/xact.h b/src/include/access/xact.h index b2bc10ee041..e27991036d4 100644 --- a/src/include/access/xact.h +++ b/src/include/access/xact.h @@ -81,6 +81,8 @@ typedef enum /* Synchronous commit level */ extern PGDLLIMPORT int synchronous_commit; +/* Allow cancelation of queries waiting for sync replication but commited locally */ +extern bool synchronous_commit_cancelation; /* used during logical streaming of a transaction */ extern PGDLLIMPORT TransactionId CheckXidAlive; From 7b43c5bfad9f0ac8b3c68daa11a219e75c55c68f Mon Sep 17 00:00:00 2001 From: reshke Date: Tue, 13 Apr 2021 16:26:33 +0300 Subject: [PATCH 08/45] Mdb-admin patch and regression tests Introduces 3 functions: extern bool mdb_admin_allow_bypass_owner_checks(Oid userId, Oid ownerId); extern void check_mdb_admin_is_member_of_role(Oid member, Oid role); extern bool mdb_admin_is_member_of_role(Oid member, Oid role); To check mdb admin belongship and role-to-role ownership transfer correctness. Our mdb_admin ACL model is the following: * Any roles user or/and roles can be granted with mdb_admin * mdb_admin member can tranfser ownershup of relations, namespaces and functions to other roles, if target role in neither: superuser, pg_read_server_files, pg_write_server_files nor pg_execute_server_program. * Allow mdb_admin to create LEAKPROOF functions * mdb admin sets session replication role * [MDB-16648 + MDB-17910]: Allow mdb admin to kill specific superuser queries MDB-27288: allow mdb_admin to kill autovac + tests mdb-27228: fix expected output One more fix Fix warns --- src/backend/catalog/namespace.c | 19 +- src/backend/commands/alter.c | 11 +- src/backend/commands/functioncmds.c | 20 ++- src/backend/commands/schemacmds.c | 19 +- src/backend/commands/tablecmds.c | 16 +- src/backend/storage/ipc/signalfuncs.c | 4 + src/backend/utils/adt/acl.c | 163 +++++++++++++++++- src/include/utils/acl.h | 9 + src/test/Makefile | 2 +- src/test/mdb_admin/.gitignore | 2 + src/test/mdb_admin/Makefile | 23 +++ src/test/mdb_admin/t/signals.pl | 75 ++++++++ .../regress/expected/create_function_sql.out | 4 +- src/test/regress/expected/mdb_admin.out | 100 +++++++++++ src/test/regress/expected/test_setup.out | 1 + src/test/regress/parallel_schedule | 4 + src/test/regress/sql/mdb_admin.sql | 87 ++++++++++ src/test/regress/sql/test_setup.sql | 2 + 18 files changed, 536 insertions(+), 25 deletions(-) create mode 100644 src/test/mdb_admin/.gitignore create mode 100644 src/test/mdb_admin/Makefile create mode 100644 src/test/mdb_admin/t/signals.pl create mode 100644 src/test/regress/expected/mdb_admin.out create mode 100644 src/test/regress/sql/mdb_admin.sql diff --git a/src/backend/catalog/namespace.c b/src/backend/catalog/namespace.c index dd5d241dfe2..10922767073 100644 --- a/src/backend/catalog/namespace.c +++ b/src/backend/catalog/namespace.c @@ -3390,6 +3390,8 @@ LookupExplicitNamespace(const char *nspname, bool missing_ok) Oid namespaceId; AclResult aclresult; + HeapTuple tuple; + Oid ownerId; /* check for pg_temp alias */ if (strcmp(nspname, "pg_temp") == 0) { @@ -3407,7 +3409,22 @@ LookupExplicitNamespace(const char *nspname, bool missing_ok) if (missing_ok && !OidIsValid(namespaceId)) return InvalidOid; - aclresult = object_aclcheck(NamespaceRelationId, namespaceId, GetUserId(), ACL_USAGE); + tuple = SearchSysCache1(NAMESPACEOID, ObjectIdGetDatum(namespaceId)); + if (!HeapTupleIsValid(tuple)) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_SCHEMA), + errmsg("schema with OID %u does not exist", namespaceId))); + + ownerId = ((Form_pg_namespace) GETSTRUCT(tuple))->nspowner; + + ReleaseSysCache(tuple); + + if (!mdb_admin_allow_bypass_owner_checks(GetUserId(), ownerId)) { + aclresult = object_aclcheck(NamespaceRelationId, namespaceId, GetUserId(), ACL_USAGE); + } else { + aclresult = ACLCHECK_OK; + } + if (aclresult != ACLCHECK_OK) aclcheck_error(aclresult, OBJECT_SCHEMA, nspname); diff --git a/src/backend/commands/alter.c b/src/backend/commands/alter.c index c801c869c1c..6624a536fca 100644 --- a/src/backend/commands/alter.c +++ b/src/backend/commands/alter.c @@ -973,7 +973,8 @@ AlterObjectOwner_internal(Oid classId, Oid objectId, Oid new_ownerId) if (!superuser()) { /* must be owner */ - if (!has_privs_of_role(GetUserId(), old_ownerId)) + if (!has_privs_of_role(GetUserId(), old_ownerId) + && !mdb_admin_allow_bypass_owner_checks(GetUserId(), old_ownerId)) { char *objname; char namebuf[NAMEDATALEN]; @@ -994,14 +995,16 @@ AlterObjectOwner_internal(Oid classId, Oid objectId, Oid new_ownerId) get_object_type(catalogId, objectId), objname); } - /* Must be able to become new owner */ - check_can_set_role(GetUserId(), new_ownerId); + + if (!mdb_admin_is_member_of_role(GetUserId(), new_ownerId)) { + /* Must be able to become new owner */ + check_can_set_role(GetUserId(), new_ownerId); + } /* New owner must have CREATE privilege on namespace */ if (OidIsValid(namespaceId)) { AclResult aclresult; - aclresult = object_aclcheck(NamespaceRelationId, namespaceId, new_ownerId, ACL_CREATE); if (aclresult != ACLCHECK_OK) diff --git a/src/backend/commands/functioncmds.c b/src/backend/commands/functioncmds.c index 0335e982b31..faf4ec08774 100644 --- a/src/backend/commands/functioncmds.c +++ b/src/backend/commands/functioncmds.c @@ -1145,9 +1145,13 @@ CreateFunction(ParseState *pstate, CreateFunctionStmt *stmt) * by security barrier views or row-level security policies. */ if (isLeakProof && !superuser()) - ereport(ERROR, - (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), - errmsg("only superuser can define a leakproof function"))); + { + Oid role = get_role_oid("mdb_admin", true); + if (!is_member_of_role(GetUserId(), role)) + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("only superuser or mdb_admin can define a leakproof function"))); + } if (transformDefElem) { @@ -1432,9 +1436,13 @@ AlterFunction(ParseState *pstate, AlterFunctionStmt *stmt) { procForm->proleakproof = boolVal(leakproof_item->arg); if (procForm->proleakproof && !superuser()) - ereport(ERROR, - (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), - errmsg("only superuser can define a leakproof function"))); + { + Oid role = get_role_oid("mdb_admin", true); + if (!is_member_of_role(GetUserId(), role)) + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("only superuser or mdb_admin can define a leakproof function"))); + } } if (cost_item) { diff --git a/src/backend/commands/schemacmds.c b/src/backend/commands/schemacmds.c index 546160f0941..d539192133d 100644 --- a/src/backend/commands/schemacmds.c +++ b/src/backend/commands/schemacmds.c @@ -383,12 +383,16 @@ AlterSchemaOwner_internal(HeapTuple tup, Relation rel, Oid newOwnerId) AclResult aclresult; /* Otherwise, must be owner of the existing object */ - if (!object_ownercheck(NamespaceRelationId, nspForm->oid, GetUserId())) + if (!mdb_admin_allow_bypass_owner_checks(GetUserId(), nspForm->nspowner) + && !object_ownercheck(NamespaceRelationId, nspForm->oid, GetUserId())) aclcheck_error(ACLCHECK_NOT_OWNER, OBJECT_SCHEMA, NameStr(nspForm->nspname)); - /* Must be able to become new owner */ - check_can_set_role(GetUserId(), newOwnerId); + + if (!mdb_admin_is_member_of_role(GetUserId(), newOwnerId)) { + /* Must be able to become new owner */ + check_can_set_role(GetUserId(), newOwnerId); + } /* * must have create-schema rights @@ -399,8 +403,14 @@ AlterSchemaOwner_internal(HeapTuple tup, Relation rel, Oid newOwnerId) * schemas. Because superusers will always have this right, we need * no special case for them. */ - aclresult = object_aclcheck(DatabaseRelationId, MyDatabaseId, GetUserId(), + + if (mdb_admin_allow_bypass_owner_checks(GetUserId(), nspForm->nspowner)) { + aclresult = ACLCHECK_OK; + } else { + aclresult = object_aclcheck(DatabaseRelationId, MyDatabaseId, GetUserId(), ACL_CREATE); + } + if (aclresult != ACLCHECK_OK) aclcheck_error(aclresult, OBJECT_DATABASE, get_database_name(MyDatabaseId)); @@ -431,7 +441,6 @@ AlterSchemaOwner_internal(HeapTuple tup, Relation rel, Oid newOwnerId) CatalogTupleUpdate(rel, &newtuple->t_self, newtuple); heap_freetuple(newtuple); - /* Update owner dependency reference */ changeDependencyOnOwner(NamespaceRelationId, nspForm->oid, newOwnerId); diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index 7f62e89342e..dbcecc2c7cb 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -16197,13 +16197,18 @@ ATExecChangeOwner(Oid relationOid, Oid newOwnerId, bool recursing, LOCKMODE lock AclResult aclresult; /* Otherwise, must be owner of the existing object */ - if (!object_ownercheck(RelationRelationId, relationOid, GetUserId())) + + if (!mdb_admin_allow_bypass_owner_checks(GetUserId(), tuple_class->relowner) + && !object_ownercheck(RelationRelationId, relationOid, GetUserId())) aclcheck_error(ACLCHECK_NOT_OWNER, get_relkind_objtype(get_rel_relkind(relationOid)), RelationGetRelationName(target_rel)); - /* Must be able to become new owner */ - check_can_set_role(GetUserId(), newOwnerId); + if (!mdb_admin_is_member_of_role(GetUserId(), newOwnerId)) { + /* Must be able to become new owner */ + check_can_set_role(GetUserId(), newOwnerId); + } + /* New owner must have CREATE privilege on namespace */ aclresult = object_aclcheck(NamespaceRelationId, namespaceOid, newOwnerId, ACL_CREATE); @@ -19594,7 +19599,7 @@ RangeVarCallbackForAlterRelation(const RangeVar *rv, Oid relid, Oid oldrelid, Form_pg_class classform; AclResult aclresult; char relkind; - + tuple = SearchSysCache1(RELOID, ObjectIdGetDatum(relid)); if (!HeapTupleIsValid(tuple)) return; /* concurrently dropped */ @@ -19602,7 +19607,8 @@ RangeVarCallbackForAlterRelation(const RangeVar *rv, Oid relid, Oid oldrelid, relkind = classform->relkind; /* Must own relation. */ - if (!object_ownercheck(RelationRelationId, relid, GetUserId())) + if (!mdb_admin_allow_bypass_owner_checks(GetUserId(), classform->relowner) + && !object_ownercheck(RelationRelationId, relid, GetUserId())) aclcheck_error(ACLCHECK_NOT_OWNER, get_relkind_objtype(get_rel_relkind(relid)), rv->relname); /* No system table modifications unless explicitly allowed. */ diff --git a/src/backend/storage/ipc/signalfuncs.c b/src/backend/storage/ipc/signalfuncs.c index a3a670ba247..7401d30d091 100644 --- a/src/backend/storage/ipc/signalfuncs.c +++ b/src/backend/storage/ipc/signalfuncs.c @@ -20,6 +20,7 @@ #include "miscadmin.h" #include "pgstat.h" #include "postmaster/syslogger.h" +#include "postmaster/bgworker.h" #include "storage/pmsignal.h" #include "storage/proc.h" #include "storage/procarray.h" @@ -51,6 +52,9 @@ static int pg_signal_backend(int pid, int sig) { PGPROC *proc = BackendPidGetProc(pid); + LocalPgBackendStatus *local_beentry; + + local_beentry = NULL; /* * BackendPidGetProc returns NULL if the pid isn't valid; but by the time diff --git a/src/backend/utils/adt/acl.c b/src/backend/utils/adt/acl.c index 1213f9106d5..bb759d0c2ef 100644 --- a/src/backend/utils/adt/acl.c +++ b/src/backend/utils/adt/acl.c @@ -133,6 +133,8 @@ static AclResult pg_role_aclcheck(Oid role_oid, Oid roleid, AclMode mode); static void RoleMembershipCacheCallback(Datum arg, int cacheid, uint32 hashvalue); +static bool has_privs_of_unwanted_system_role(Oid role); + /* * Test whether an identifier char can be left unquoted in ACLs. @@ -5281,6 +5283,58 @@ roles_is_member_of(Oid roleid, enum RoleRecurseType type, * * See also member_can_set_role, below. */ + +/* +* This is basically original postgresql privs-check function +*/ + +static bool +has_privs_of_role_strict(Oid member, Oid role) +{ + /* Fast path for simple case */ + if (member == role) + return true; + + /* Superusers have every privilege, so are part of every role */ + if (superuser_arg(member)) + return true; + + /* + * Find all the roles that member has the privileges of, including + * multi-level recursion, then see if target role is any one of them. + */ + return list_member_oid(roles_is_member_of(member, ROLERECURSE_PRIVS, + InvalidOid, NULL), + role); +} + +/* +* Check that role is either one of "dangerous" system role +* or has "strict" (not through mdb_admin) +* privs of this role +*/ + +static bool +has_privs_of_unwanted_system_role(Oid role) { + if (has_privs_of_role_strict(role, ROLE_PG_READ_SERVER_FILES)) { + return true; + } + if (has_privs_of_role_strict(role, ROLE_PG_WRITE_SERVER_FILES)) { + return true; + } + if (has_privs_of_role_strict(role, ROLE_PG_EXECUTE_SERVER_PROGRAM)) { + return true; + } + if (has_privs_of_role_strict(role, ROLE_PG_READ_ALL_DATA)) { + return true; + } + if (has_privs_of_role_strict(role, ROLE_PG_WRITE_ALL_DATA)) { + return true; + } + + return false; +} + bool has_privs_of_role(Oid member, Oid role) { @@ -5301,6 +5355,47 @@ has_privs_of_role(Oid member, Oid role) role); } +// -- non-upstream patch begin +/* + * Is userId allowed to bypass ownership check + * and tranfer onwership to ownerId role? + */ +bool +mdb_admin_allow_bypass_owner_checks(Oid userId, Oid ownerId) +{ + Oid mdb_admin_roleoid; + /* + * Never allow nobody to grant objects to + * superusers. + * This can result in various CVE. + * For paranoic reasons, check this even before + * membership of mdb_admin role. + */ + if (superuser_arg(ownerId)) { + return false; + } + + mdb_admin_roleoid = get_role_oid("mdb_admin", true /*if nodoby created mdb_admin role in this database*/); + /* Is userId actually member of mdb admin? */ + if (!is_member_of_role(userId, mdb_admin_roleoid)) { + /* if no, disallow. */ + return false; + } + + /* + * Now, we need to check if ownerId + * is some dangerous role to trasfer membership to. + * + * For now, we check that ownerId does not have + * priviledge to execute server program or/and + * read/write server files, or/and pg read/write all data + */ + + /* All checks passed, hope will not be hacked here (again) */ + return !has_privs_of_unwanted_system_role(ownerId); +} + +// -- non-upstream patch end /* * Can member use SET ROLE to this role? * @@ -5348,6 +5443,73 @@ check_can_set_role(Oid member, Oid role) GetUserNameFromId(role, false)))); } +// -- mdb admin patch +/* + * check_mdb_admin_is_member_of_role + * is_member_of_role with a standard permission-violation error if not in usual case + * Is case `member` in mdb_admin we check that role is neither of superuser, pg_read/write + * server files nor pg_execute_server_program or pg_read/write all data + */ +void +check_mdb_admin_is_member_of_role(Oid member, Oid role) +{ + Oid mdb_admin_roleoid; + /* fast path - if we are superuser, its ok */ + if (superuser_arg(member)) { + return; + } + + mdb_admin_roleoid = get_role_oid("mdb_admin", true /*if nodoby created mdb_admin role in this database*/); + /* Is userId actually member of mdb admin? */ + if (is_member_of_role(member, mdb_admin_roleoid)) { + + /* role is mdb admin */ + if (superuser_arg(role)) { + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("cannot transfer ownership to superuser \"%s\"", + GetUserNameFromId(role, false)))); + } + + if (has_privs_of_unwanted_system_role(role)) { + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("forbidden to transfer ownership to this system role in Cloud"))); + } + } else { + /* if no, check membership transfer in usual way. */ + + if (!is_member_of_role(member, role)) { + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("must be member of role \"%s\"", + GetUserNameFromId(role, false)))); + } + } +} + +bool mdb_admin_is_member_of_role(Oid member, Oid role) { + Oid mdb_admin_roleoid; + /* fast path - if we are superuser, its ok */ + if (superuser_arg(member)) { + return true; + } + + mdb_admin_roleoid = get_role_oid("mdb_admin", true /*if nodoby created mdb_admin role in this database*/); + /* Is userId actually member of mdb admin? */ + if (!is_member_of_role(member, mdb_admin_roleoid)) { + return false; + } + /* role is mdb admin */ + if (superuser_arg(role)) { + return false; + } + + return !has_privs_of_unwanted_system_role(role); +} + +// -- mdb admin patch + /* * Is member a member of role (directly or indirectly)? * @@ -5504,7 +5666,6 @@ select_best_grantor(Oid roleId, AclMode privileges, */ roles_list = roles_is_member_of(roleId, ROLERECURSE_PRIVS, InvalidOid, NULL); - /* initialize candidate result as default */ *grantorId = roleId; *grantOptions = ACL_NO_RIGHTS; diff --git a/src/include/utils/acl.h b/src/include/utils/acl.h index 01ae5b719fd..7cfd92da9dc 100644 --- a/src/include/utils/acl.h +++ b/src/include/utils/acl.h @@ -215,6 +215,15 @@ extern void check_can_set_role(Oid member, Oid role); extern bool is_member_of_role(Oid member, Oid role); extern bool is_member_of_role_nosuper(Oid member, Oid role); extern bool is_admin_of_role(Oid member, Oid role); + +// -- non-upstream patch begin +extern bool mdb_admin_allow_bypass_owner_checks(Oid userId, Oid ownerId); + +extern void check_mdb_admin_is_member_of_role(Oid member, Oid role); + +extern bool mdb_admin_is_member_of_role(Oid member, Oid role); +// -- non-upstream patch end + extern Oid select_best_admin(Oid member, Oid role); extern Oid get_role_oid(const char *rolname, bool missing_ok); extern Oid get_role_oid_or_public(const char *rolname); diff --git a/src/test/Makefile b/src/test/Makefile index 511a72e6238..d1c73b44c70 100644 --- a/src/test/Makefile +++ b/src/test/Makefile @@ -12,7 +12,7 @@ subdir = src/test top_builddir = ../.. include $(top_builddir)/src/Makefile.global -SUBDIRS = perl postmaster regress isolation modules authentication recovery subscription +SUBDIRS = perl postmaster regress isolation modules authentication recovery subscription mdb_admin ifeq ($(with_icu),yes) SUBDIRS += icu diff --git a/src/test/mdb_admin/.gitignore b/src/test/mdb_admin/.gitignore new file mode 100644 index 00000000000..871e943d50e --- /dev/null +++ b/src/test/mdb_admin/.gitignore @@ -0,0 +1,2 @@ +# Generated by test suite +/tmp_check/ diff --git a/src/test/mdb_admin/Makefile b/src/test/mdb_admin/Makefile new file mode 100644 index 00000000000..e4e82367da9 --- /dev/null +++ b/src/test/mdb_admin/Makefile @@ -0,0 +1,23 @@ +#------------------------------------------------------------------------- +# +# Makefile for src/test/mdb_admin +# +# Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group +# Portions Copyright (c) 1994, Regents of the University of California +# +# src/test/mdb_admin/Makefile +# +#------------------------------------------------------------------------- + +subdir = src/test/mdb_admin +top_builddir = ../../.. +include $(top_builddir)/src/Makefile.global + +check: + $(prove_check) + +installcheck: + $(prove_installcheck) + +clean distclean maintainer-clean: + rm -rf tmp_check diff --git a/src/test/mdb_admin/t/signals.pl b/src/test/mdb_admin/t/signals.pl new file mode 100644 index 00000000000..25552092e94 --- /dev/null +++ b/src/test/mdb_admin/t/signals.pl @@ -0,0 +1,75 @@ + +# Copyright (c) 2024-2024, MDB, Mother Russia + +# Minimal test testing streaming replication +use strict; +use warnings; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +# Initialize primary node +my $node_primary = PostgreSQL::Test::Cluster->new('primary'); +$node_primary->init(); +$node_primary->start; + +# Create some content on primary and check its presence in standby nodes +$node_primary->safe_psql('postgres', + " + CREATE DATABASE regress; + CREATE ROLE mdb_admin; + GRANT pg_signal_autovacuum_worker TO mdb_admin; + CREATE ROLE mdb_reg_lh_1; + CREATE ROLE mdb_reg_lh_2; + GRANT pg_signal_backend TO mdb_admin; + GRANT pg_signal_backend TO mdb_reg_lh_1; + GRANT mdb_admin TO mdb_reg_lh_2; +"); + +# Create some content on primary and check its presence in standby nodes +$node_primary->safe_psql('regress', + " + CREATE TABLE tab_int(i int); + INSERT INTO tab_int SELECT * FROm generate_series(1, 1000000); + ALTER SYSTEM SET autovacuum_vacuum_cost_limit TO 1; + ALTER SYSTEM SET autovacuum_vacuum_cost_delay TO 100; + ALTER SYSTEM SET autovacuum_naptime TO 1; +"); + +$node_primary->restart; + +sleep 1; + +my $res_pid = $node_primary->safe_psql('regress', + " + SELECT pid FROM pg_stat_activity WHERE backend_type = 'autovacuum worker' and datname = 'regress'; +"); + + +print "pid is $res_pid\n"; + +ok(1); + + +my ($res_reg_lh_1, $stdout_reg_lh_1, $stderr_reg_lh_1) = $node_primary->psql('regress', + " + SET ROLE mdb_reg_lh_1; + SELECT pg_terminate_backend($res_pid); +"); + +# print ($res_reg_lh_1, $stdout_reg_lh_1, $stderr_reg_lh_1, "\n"); + +ok($res_reg_lh_1 != 0, "should fail for non-mdb_admin"); +like($stderr_reg_lh_1, qr/Only roles with privileges of the "pg_signal_autovacuum_worker" role may terminate autovacuum workers./, "matches"); + +my ($res_reg_lh_2, $stdout_reg_lh_2, $stderr_reg_lh_2) = $node_primary->psql('regress', + " + SET ROLE mdb_reg_lh_2; + SELECT pg_terminate_backend($res_pid); +"); + +ok($res_reg_lh_2 == 0, "should success for mdb_admin"); + +# print ($res_reg_lh_2, $stdout_reg_lh_2, $stderr_reg_lh_2, "\n"); + +done_testing(); diff --git a/src/test/regress/expected/create_function_sql.out b/src/test/regress/expected/create_function_sql.out index da112608d66..fc10a9606d2 100644 --- a/src/test/regress/expected/create_function_sql.out +++ b/src/test/regress/expected/create_function_sql.out @@ -166,10 +166,10 @@ SET SESSION AUTHORIZATION regress_unpriv_user; SET search_path TO temp_func_test, public; ALTER FUNCTION functest_E_1(int) NOT LEAKPROOF; ALTER FUNCTION functest_E_2(int) LEAKPROOF; -ERROR: only superuser can define a leakproof function +ERROR: only superuser or mdb_admin can define a leakproof function CREATE FUNCTION functest_E_3(int) RETURNS bool LANGUAGE 'sql' LEAKPROOF AS 'SELECT $1 < 200'; -- fail -ERROR: only superuser can define a leakproof function +ERROR: only superuser or mdb_admin can define a leakproof function RESET SESSION AUTHORIZATION; -- -- CALLED ON NULL INPUT | RETURNS NULL ON NULL INPUT | STRICT diff --git a/src/test/regress/expected/mdb_admin.out b/src/test/regress/expected/mdb_admin.out new file mode 100644 index 00000000000..20722750c05 --- /dev/null +++ b/src/test/regress/expected/mdb_admin.out @@ -0,0 +1,100 @@ +CREATE ROLE regress_mdb_admin_user1; +CREATE ROLE regress_mdb_admin_user2; +CREATE ROLE regress_mdb_admin_user3; +CREATE ROLE regress_superuser WITH SUPERUSER; +GRANT mdb_admin TO regress_mdb_admin_user1; +GRANT CREATE ON DATABASE regression TO regress_mdb_admin_user2; +GRANT CREATE ON DATABASE regression TO regress_mdb_admin_user3; +-- mdb admin trasfers ownership to another role +SET ROLE regress_mdb_admin_user2; +CREATE FUNCTION regress_mdb_admin_add(integer, integer) RETURNS integer + AS 'SELECT $1 + $2;' + LANGUAGE SQL + IMMUTABLE + RETURNS NULL ON NULL INPUT; +CREATE SCHEMA regress_mdb_admin_schema; +GRANT CREATE ON SCHEMA regress_mdb_admin_schema TO regress_mdb_admin_user3; +CREATE TABLE regress_mdb_admin_schema.regress_mdb_admin_table(); +CREATE TABLE regress_mdb_admin_table(); +CREATE VIEW regress_mdb_admin_view as SELECT 1; +SET ROLE regress_mdb_admin_user1; +ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO regress_mdb_admin_user3; +ALTER VIEW regress_mdb_admin_view OWNER TO regress_mdb_admin_user3; +ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO regress_mdb_admin_user3; +ALTER TABLE regress_mdb_admin_table OWNER TO regress_mdb_admin_user3; +ALTER SCHEMA regress_mdb_admin_schema OWNER TO regress_mdb_admin_user3; +-- mdb admin fails to transfer ownership to superusers and particular system roles +ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO regress_superuser; +ERROR: must be able to SET ROLE "regress_superuser" +ALTER VIEW regress_mdb_admin_view OWNER TO regress_superuser; +ERROR: must be able to SET ROLE "regress_superuser" +ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO regress_superuser; +ERROR: must be able to SET ROLE "regress_superuser" +ALTER TABLE regress_mdb_admin_table OWNER TO regress_superuser; +ERROR: must be able to SET ROLE "regress_superuser" +ALTER SCHEMA regress_mdb_admin_schema OWNER TO regress_superuser; +ERROR: must be able to SET ROLE "regress_superuser" +ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO pg_execute_server_program; +ERROR: must be able to SET ROLE "pg_execute_server_program" +ALTER VIEW regress_mdb_admin_view OWNER TO pg_execute_server_program; +ERROR: must be able to SET ROLE "pg_execute_server_program" +ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO pg_execute_server_program; +ERROR: must be able to SET ROLE "pg_execute_server_program" +ALTER TABLE regress_mdb_admin_table OWNER TO pg_execute_server_program; +ERROR: must be able to SET ROLE "pg_execute_server_program" +ALTER SCHEMA regress_mdb_admin_schema OWNER TO pg_execute_server_program; +ERROR: must be able to SET ROLE "pg_execute_server_program" +ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO pg_write_server_files; +ERROR: must be able to SET ROLE "pg_write_server_files" +ALTER VIEW regress_mdb_admin_view OWNER TO pg_write_server_files; +ERROR: must be able to SET ROLE "pg_write_server_files" +ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO pg_write_server_files; +ERROR: must be able to SET ROLE "pg_write_server_files" +ALTER TABLE regress_mdb_admin_table OWNER TO pg_write_server_files; +ERROR: must be able to SET ROLE "pg_write_server_files" +ALTER SCHEMA regress_mdb_admin_schema OWNER TO pg_write_server_files; +ERROR: must be able to SET ROLE "pg_write_server_files" +ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO pg_read_server_files; +ERROR: must be able to SET ROLE "pg_read_server_files" +ALTER VIEW regress_mdb_admin_view OWNER TO pg_read_server_files; +ERROR: must be able to SET ROLE "pg_read_server_files" +ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO pg_read_server_files; +ERROR: must be able to SET ROLE "pg_read_server_files" +ALTER TABLE regress_mdb_admin_table OWNER TO pg_read_server_files; +ERROR: must be able to SET ROLE "pg_read_server_files" +ALTER SCHEMA regress_mdb_admin_schema OWNER TO pg_read_server_files; +ERROR: must be able to SET ROLE "pg_read_server_files" +ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO pg_write_all_data; +ERROR: must be able to SET ROLE "pg_write_all_data" +ALTER VIEW regress_mdb_admin_view OWNER TO pg_write_all_data; +ERROR: must be able to SET ROLE "pg_write_all_data" +ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO pg_write_all_data; +ERROR: must be able to SET ROLE "pg_write_all_data" +ALTER TABLE regress_mdb_admin_table OWNER TO pg_write_all_data; +ERROR: must be able to SET ROLE "pg_write_all_data" +ALTER SCHEMA regress_mdb_admin_schema OWNER TO pg_write_all_data; +ERROR: must be able to SET ROLE "pg_write_all_data" +ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO pg_read_all_data; +ERROR: must be able to SET ROLE "pg_read_all_data" +ALTER VIEW regress_mdb_admin_view OWNER TO pg_read_all_data; +ERROR: must be able to SET ROLE "pg_read_all_data" +ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO pg_read_all_data; +ERROR: must be able to SET ROLE "pg_read_all_data" +ALTER TABLE regress_mdb_admin_table OWNER TO pg_read_all_data; +ERROR: must be able to SET ROLE "pg_read_all_data" +ALTER SCHEMA regress_mdb_admin_schema OWNER TO pg_read_all_data; +ERROR: must be able to SET ROLE "pg_read_all_data" +-- end tests +RESET SESSION AUTHORIZATION; +-- +REVOKE CREATE ON DATABASE regression FROM regress_mdb_admin_user2; +REVOKE CREATE ON DATABASE regression FROM regress_mdb_admin_user3; +DROP VIEW regress_mdb_admin_view; +DROP FUNCTION regress_mdb_admin_add; +DROP TABLE regress_mdb_admin_schema.regress_mdb_admin_table; +DROP TABLE regress_mdb_admin_table; +DROP SCHEMA regress_mdb_admin_schema; +DROP ROLE regress_mdb_admin_user1; +DROP ROLE regress_mdb_admin_user2; +DROP ROLE regress_mdb_admin_user3; +DROP ROLE regress_superuser; diff --git a/src/test/regress/expected/test_setup.out b/src/test/regress/expected/test_setup.out index 93a4c2691c1..b6ace34542b 100644 --- a/src/test/regress/expected/test_setup.out +++ b/src/test/regress/expected/test_setup.out @@ -235,3 +235,4 @@ create function fipshash(text) returns text strict immutable parallel safe leakproof return substr(encode(sha256($1::bytea), 'hex'), 1, 32); +CREATE ROLE mdb_admin; diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule index e1e0c540194..afb347ec600 100644 --- a/src/test/regress/parallel_schedule +++ b/src/test/regress/parallel_schedule @@ -11,6 +11,10 @@ # required setup steps test: test_setup +# mdb admin simple checks + +test: mdb_admin + # ---------- # The first group of parallel tests # ---------- diff --git a/src/test/regress/sql/mdb_admin.sql b/src/test/regress/sql/mdb_admin.sql new file mode 100644 index 00000000000..b6b048e5692 --- /dev/null +++ b/src/test/regress/sql/mdb_admin.sql @@ -0,0 +1,87 @@ +CREATE ROLE regress_mdb_admin_user1; +CREATE ROLE regress_mdb_admin_user2; +CREATE ROLE regress_mdb_admin_user3; + +CREATE ROLE regress_superuser WITH SUPERUSER; + +GRANT mdb_admin TO regress_mdb_admin_user1; +GRANT CREATE ON DATABASE regression TO regress_mdb_admin_user2; +GRANT CREATE ON DATABASE regression TO regress_mdb_admin_user3; + +-- mdb admin trasfers ownership to another role + +SET ROLE regress_mdb_admin_user2; +CREATE FUNCTION regress_mdb_admin_add(integer, integer) RETURNS integer + AS 'SELECT $1 + $2;' + LANGUAGE SQL + IMMUTABLE + RETURNS NULL ON NULL INPUT; + +CREATE SCHEMA regress_mdb_admin_schema; +GRANT CREATE ON SCHEMA regress_mdb_admin_schema TO regress_mdb_admin_user3; +CREATE TABLE regress_mdb_admin_schema.regress_mdb_admin_table(); +CREATE TABLE regress_mdb_admin_table(); +CREATE VIEW regress_mdb_admin_view as SELECT 1; +SET ROLE regress_mdb_admin_user1; + +ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO regress_mdb_admin_user3; +ALTER VIEW regress_mdb_admin_view OWNER TO regress_mdb_admin_user3; +ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO regress_mdb_admin_user3; +ALTER TABLE regress_mdb_admin_table OWNER TO regress_mdb_admin_user3; +ALTER SCHEMA regress_mdb_admin_schema OWNER TO regress_mdb_admin_user3; + + +-- mdb admin fails to transfer ownership to superusers and particular system roles + +ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO regress_superuser; +ALTER VIEW regress_mdb_admin_view OWNER TO regress_superuser; +ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO regress_superuser; +ALTER TABLE regress_mdb_admin_table OWNER TO regress_superuser; +ALTER SCHEMA regress_mdb_admin_schema OWNER TO regress_superuser; + +ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO pg_execute_server_program; +ALTER VIEW regress_mdb_admin_view OWNER TO pg_execute_server_program; +ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO pg_execute_server_program; +ALTER TABLE regress_mdb_admin_table OWNER TO pg_execute_server_program; +ALTER SCHEMA regress_mdb_admin_schema OWNER TO pg_execute_server_program; + +ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO pg_write_server_files; +ALTER VIEW regress_mdb_admin_view OWNER TO pg_write_server_files; +ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO pg_write_server_files; +ALTER TABLE regress_mdb_admin_table OWNER TO pg_write_server_files; +ALTER SCHEMA regress_mdb_admin_schema OWNER TO pg_write_server_files; + +ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO pg_read_server_files; +ALTER VIEW regress_mdb_admin_view OWNER TO pg_read_server_files; +ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO pg_read_server_files; +ALTER TABLE regress_mdb_admin_table OWNER TO pg_read_server_files; +ALTER SCHEMA regress_mdb_admin_schema OWNER TO pg_read_server_files; + +ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO pg_write_all_data; +ALTER VIEW regress_mdb_admin_view OWNER TO pg_write_all_data; +ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO pg_write_all_data; +ALTER TABLE regress_mdb_admin_table OWNER TO pg_write_all_data; +ALTER SCHEMA regress_mdb_admin_schema OWNER TO pg_write_all_data; + +ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO pg_read_all_data; +ALTER VIEW regress_mdb_admin_view OWNER TO pg_read_all_data; +ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO pg_read_all_data; +ALTER TABLE regress_mdb_admin_table OWNER TO pg_read_all_data; +ALTER SCHEMA regress_mdb_admin_schema OWNER TO pg_read_all_data; + +-- end tests + +RESET SESSION AUTHORIZATION; +-- +REVOKE CREATE ON DATABASE regression FROM regress_mdb_admin_user2; +REVOKE CREATE ON DATABASE regression FROM regress_mdb_admin_user3; + +DROP VIEW regress_mdb_admin_view; +DROP FUNCTION regress_mdb_admin_add; +DROP TABLE regress_mdb_admin_schema.regress_mdb_admin_table; +DROP TABLE regress_mdb_admin_table; +DROP SCHEMA regress_mdb_admin_schema; +DROP ROLE regress_mdb_admin_user1; +DROP ROLE regress_mdb_admin_user2; +DROP ROLE regress_mdb_admin_user3; +DROP ROLE regress_superuser; diff --git a/src/test/regress/sql/test_setup.sql b/src/test/regress/sql/test_setup.sql index 5854399a028..ed46cf1f2eb 100644 --- a/src/test/regress/sql/test_setup.sql +++ b/src/test/regress/sql/test_setup.sql @@ -289,3 +289,5 @@ create function fipshash(text) returns text strict immutable parallel safe leakproof return substr(encode(sha256($1::bytea), 'hex'), 1, 32); + +CREATE ROLE mdb_admin; From 9300f63a658899f6d6c4593d45fc7ca0b5f18832 Mon Sep 17 00:00:00 2001 From: Andrey Borodin Date: Sun, 6 Dec 2020 10:01:59 +0500 Subject: [PATCH 09/45] MDB replication role patch MDB replication role regression tests Patch allows user with mdb_replication role to use pg_create_logical_replication_slot, pg_replication_slot_advance, pg_drop_replication_slot functions to manage logical replication slots. Also, users with mdb_admin (which is memner of pg_create_subscription) can create subscribptions. Slot names starting with MDB.* are forbidden. Add run as owner tap tests More test cases in mdb_102 Fix test Fix tests once again Never check for superuser in walsender --- .../test_decoding/expected/permissions.out | 94 ++++- contrib/test_decoding/sql/permissions.sql | 29 ++ src/backend/commands/subscriptioncmds.c | 10 +- src/backend/postmaster/syslogger.c | 2 + .../replication/logical/logicalfuncs.c | 7 +- src/backend/replication/logical/relation.c | 7 + src/backend/replication/logical/tablesync.c | 7 + src/backend/replication/logical/worker.c | 31 +- src/backend/replication/slot.c | 38 ++ src/backend/replication/slotfuncs.c | 15 +- src/backend/replication/walsender.c | 28 ++ src/backend/utils/init/postinit.c | 14 +- src/include/replication/slot.h | 27 ++ src/include/replication/walsender.h | 4 + .../t/101_logical_decoding_mdb_replication.pl | 159 +++++++++ src/test/regress/expected/mdb_replication.out | 35 ++ src/test/regress/expected/test_setup.out | 3 + src/test/regress/parallel_schedule | 4 + src/test/regress/sql/mdb_replication.sql | 41 +++ src/test/regress/sql/test_setup.sql | 4 + .../t/101_mdb_rep_changes_nonsuperuser.pl | 336 ++++++++++++++++++ .../t/102_mdb_check_permissions.pl | 202 +++++++++++ 22 files changed, 1081 insertions(+), 16 deletions(-) create mode 100644 src/test/recovery/t/101_logical_decoding_mdb_replication.pl create mode 100644 src/test/regress/expected/mdb_replication.out create mode 100644 src/test/regress/sql/mdb_replication.sql create mode 100644 src/test/subscription/t/101_mdb_rep_changes_nonsuperuser.pl create mode 100644 src/test/subscription/t/102_mdb_check_permissions.pl diff --git a/contrib/test_decoding/expected/permissions.out b/contrib/test_decoding/expected/permissions.out index 8d100646ce6..712679e3739 100644 --- a/contrib/test_decoding/expected/permissions.out +++ b/contrib/test_decoding/expected/permissions.out @@ -2,6 +2,9 @@ SET synchronous_commit = on; -- setup CREATE ROLE regress_lr_normal; +CREATE ROLE mdb_replication; +CREATE ROLE regress_lr_mdb_replication; +GRANT mdb_replication TO regress_lr_mdb_replication; CREATE ROLE regress_lr_superuser SUPERUSER; CREATE ROLE regress_lr_replication REPLICATION; CREATE TABLE lr_test(data text); @@ -28,6 +31,24 @@ SELECT pg_drop_replication_slot('regression_slot'); (1 row) +-- superuser can control slot starts from mdb +SELECT 'init' FROM pg_create_logical_replication_slot('mdb_regression_slot', 'test_decoding'); + ?column? +---------- + init +(1 row) + +SELECT data FROM pg_logical_slot_get_changes('mdb_regression_slot', NULL, NULL, 'include-xids', '0', 'skip-empty-xacts', '1'); + data +------ +(0 rows) + +SELECT pg_drop_replication_slot('mdb_regression_slot'); + pg_drop_replication_slot +-------------------------- + +(1 row) + RESET ROLE; -- replication user can control replication SET ROLE regress_lr_replication; @@ -50,20 +71,85 @@ SELECT pg_drop_replication_slot('regression_slot'); (1 row) +-- replication user can control slot starts from mdb +SELECT 'init' FROM pg_create_logical_replication_slot('mdb_regression_slot', 'test_decoding'); + ?column? +---------- + init +(1 row) + +SELECT data FROM pg_logical_slot_get_changes('mdb_regression_slot', NULL, NULL, 'include-xids', '0', 'skip-empty-xacts', '1'); + data +------ +(0 rows) + +SELECT pg_drop_replication_slot('mdb_regression_slot'); + pg_drop_replication_slot +-------------------------- + +(1 row) + +SELECT 'init' FROM pg_create_logical_replication_slot('mdb_regression_slot', 'test_decoding'); + ?column? +---------- + init +(1 row) + +RESET ROLE; +-- mdb_replication user can control replication +SET ROLE regress_lr_mdb_replication; +SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'test_decoding'); + ?column? +---------- + init +(1 row) + +INSERT INTO lr_test VALUES('lr_mdb_replication_init'); +ERROR: permission denied for table lr_test +SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'include-xids', '0', 'skip-empty-xacts', '1'); + data +------ +(0 rows) + +SELECT pg_drop_replication_slot('regression_slot'); + pg_drop_replication_slot +-------------------------- + +(1 row) + +-- mdb_replication user can't control slot starts from mdb +SELECT 'init' FROM pg_create_logical_replication_slot('mdb_regression_slot2', 'test_decoding'); +ERROR: slot name "mdb_regression_slot2" is reserved +DETAIL: Slot names starting with "mdb" are reserved. +SELECT data FROM pg_logical_slot_get_changes('mdb_regression_slot', NULL, NULL, 'include-xids', '0', 'skip-empty-xacts', '1'); +ERROR: slot name "mdb_regression_slot" is reserved +DETAIL: Slot names starting with "mdb" are reserved. +SELECT pg_drop_replication_slot('mdb_regression_slot'); +ERROR: slot name "mdb_regression_slot" is reserved +DETAIL: Slot names starting with "mdb" are reserved. +RESET ROLE; +-- cleanup +SET ROLE regress_lr_superuser; +SELECT pg_drop_replication_slot('mdb_regression_slot'); + pg_drop_replication_slot +-------------------------- + +(1 row) + RESET ROLE; -- plain user *can't* can control replication SET ROLE regress_lr_normal; SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'test_decoding'); ERROR: permission denied to use replication slots -DETAIL: Only roles with the REPLICATION attribute may use replication slots. +DETAIL: must be superuser, replication role or mdb_replication to use replication slots INSERT INTO lr_test VALUES('lr_superuser_init'); ERROR: permission denied for table lr_test SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'include-xids', '0', 'skip-empty-xacts', '1'); ERROR: permission denied to use replication slots -DETAIL: Only roles with the REPLICATION attribute may use replication slots. +DETAIL: must be superuser, replication role or mdb_replication to use replication slots SELECT pg_drop_replication_slot('regression_slot'); ERROR: permission denied to use replication slots -DETAIL: Only roles with the REPLICATION attribute may use replication slots. +DETAIL: must be superuser, replication role or mdb_replication to use replication slots SELECT pg_sync_replication_slots(); ERROR: permission denied to use replication slots DETAIL: Only roles with the REPLICATION attribute may use replication slots. @@ -97,7 +183,7 @@ RESET ROLE; SET ROLE regress_lr_normal; SELECT pg_drop_replication_slot('regression_slot'); ERROR: permission denied to use replication slots -DETAIL: Only roles with the REPLICATION attribute may use replication slots. +DETAIL: must be superuser, replication role or mdb_replication to use replication slots RESET ROLE; -- all users can see existing slots SET ROLE regress_lr_superuser; diff --git a/contrib/test_decoding/sql/permissions.sql b/contrib/test_decoding/sql/permissions.sql index 94db936aee2..80c91eba619 100644 --- a/contrib/test_decoding/sql/permissions.sql +++ b/contrib/test_decoding/sql/permissions.sql @@ -3,6 +3,9 @@ SET synchronous_commit = on; -- setup CREATE ROLE regress_lr_normal; +CREATE ROLE mdb_replication; +CREATE ROLE regress_lr_mdb_replication; +GRANT mdb_replication TO regress_lr_mdb_replication; CREATE ROLE regress_lr_superuser SUPERUSER; CREATE ROLE regress_lr_replication REPLICATION; CREATE TABLE lr_test(data text); @@ -13,6 +16,10 @@ SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'test_d INSERT INTO lr_test VALUES('lr_superuser_init'); SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'include-xids', '0', 'skip-empty-xacts', '1'); SELECT pg_drop_replication_slot('regression_slot'); +-- superuser can control slot starts from mdb +SELECT 'init' FROM pg_create_logical_replication_slot('mdb_regression_slot', 'test_decoding'); +SELECT data FROM pg_logical_slot_get_changes('mdb_regression_slot', NULL, NULL, 'include-xids', '0', 'skip-empty-xacts', '1'); +SELECT pg_drop_replication_slot('mdb_regression_slot'); RESET ROLE; -- replication user can control replication @@ -21,6 +28,28 @@ SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'test_d INSERT INTO lr_test VALUES('lr_superuser_init'); SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'include-xids', '0', 'skip-empty-xacts', '1'); SELECT pg_drop_replication_slot('regression_slot'); +-- replication user can control slot starts from mdb +SELECT 'init' FROM pg_create_logical_replication_slot('mdb_regression_slot', 'test_decoding'); +SELECT data FROM pg_logical_slot_get_changes('mdb_regression_slot', NULL, NULL, 'include-xids', '0', 'skip-empty-xacts', '1'); +SELECT pg_drop_replication_slot('mdb_regression_slot'); +SELECT 'init' FROM pg_create_logical_replication_slot('mdb_regression_slot', 'test_decoding'); +RESET ROLE; + +-- mdb_replication user can control replication +SET ROLE regress_lr_mdb_replication; +SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'test_decoding'); +INSERT INTO lr_test VALUES('lr_mdb_replication_init'); +SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'include-xids', '0', 'skip-empty-xacts', '1'); +SELECT pg_drop_replication_slot('regression_slot'); +-- mdb_replication user can't control slot starts from mdb +SELECT 'init' FROM pg_create_logical_replication_slot('mdb_regression_slot2', 'test_decoding'); +SELECT data FROM pg_logical_slot_get_changes('mdb_regression_slot', NULL, NULL, 'include-xids', '0', 'skip-empty-xacts', '1'); +SELECT pg_drop_replication_slot('mdb_regression_slot'); +RESET ROLE; + +-- cleanup +SET ROLE regress_lr_superuser; +SELECT pg_drop_replication_slot('mdb_regression_slot'); RESET ROLE; -- plain user *can't* can control replication diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c index b752d8b1097..255b8c1ef81 100644 --- a/src/backend/commands/subscriptioncmds.c +++ b/src/backend/commands/subscriptioncmds.c @@ -552,7 +552,7 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt, bits32 supported_opts; SubOpts opts = {0}; AclResult aclresult; - + /* * Parse and check options. * @@ -580,6 +580,7 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt, * attempts to access arbitrary network destinations, so require the user * to have been specifically authorized to create subscriptions. */ + /* MDB: mdb_admin need to be granted with pg_create_subscription role */ if (!has_privs_of_role(owner, ROLE_PG_CREATE_SUBSCRIPTION)) ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), @@ -1991,8 +1992,11 @@ AlterSubscriptionOwner_internal(Relation rel, HeapTuple tup, Oid newOwnerId) errmsg("password_required=false is superuser-only"), errhint("Subscriptions with the password_required option set to false may only be created or modified by the superuser."))); - /* Must be able to become new owner */ - check_can_set_role(GetUserId(), newOwnerId); + /* if we are mdb_admin, check that we alter ownership to unprivileged role */ + if (!mdb_admin_allow_bypass_owner_checks(GetUserId(), newOwnerId)) { + /* else we must be able to become new owner */ + check_can_set_role(GetUserId(), newOwnerId); + } /* * current owner must have CREATE on database diff --git a/src/backend/postmaster/syslogger.c b/src/backend/postmaster/syslogger.c index 50c2edec1f6..fa8231b8a64 100644 --- a/src/backend/postmaster/syslogger.c +++ b/src/backend/postmaster/syslogger.c @@ -342,6 +342,8 @@ SysLoggerMain(const void *startup_data, size_t startup_data_len) #ifndef WIN32 AddWaitEventToSet(wes, WL_SOCKET_READABLE, syslogPipe[0], NULL, NULL); #endif + fclose(stderr); + fclose(stdout); /* main worker loop */ for (;;) diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c index ca53caac2f2..415f2f22e54 100644 --- a/src/backend/replication/logical/logicalfuncs.c +++ b/src/backend/replication/logical/logicalfuncs.c @@ -28,6 +28,8 @@ #include "replication/decode.h" #include "replication/logical.h" #include "replication/message.h" +#include "storage/fd.h" +#include "utils/acl.h" #include "utils/array.h" #include "utils/builtins.h" #include "utils/inval.h" @@ -113,8 +115,7 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin List *options = NIL; DecodingOutputState *p; - CheckSlotPermissions(); - + CheckMDBReplSlotPermissions(); CheckLogicalDecodingRequirements(); if (PG_ARGISNULL(0)) @@ -123,6 +124,8 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin errmsg("slot name must not be null"))); name = PG_GETARG_NAME(0); + CheckMDBReservedName(NameStr(*name)); + if (PG_ARGISNULL(1)) upto_lsn = InvalidXLogRecPtr; else diff --git a/src/backend/replication/logical/relation.c b/src/backend/replication/logical/relation.c index f59046ad620..99962b32c48 100644 --- a/src/backend/replication/logical/relation.c +++ b/src/backend/replication/logical/relation.c @@ -20,6 +20,7 @@ #include "access/amapi.h" #include "access/genam.h" #include "access/table.h" +#include "catalog/catalog.h" #include "catalog/namespace.h" #include "catalog/pg_subscription_rel.h" #include "executor/executor.h" @@ -423,6 +424,12 @@ logicalrep_rel_open(LogicalRepRelId remoteid, LOCKMODE lockmode) entry->localrel = table_open(relid, NoLock); entry->localreloid = relid; + /* Don't allow catalog access */ + if (IsSystemClass(relid, entry->localrel->rd_rel)) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("logical replication target relation \"%s.%s\" is a system class", + remoterel->nspname, remoterel->relname))); /* Check for supported relkind. */ CheckSubscriptionRelkind(entry->localrel->rd_rel->relkind, remoterel->nspname, remoterel->relname); diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c index 49d4ec3bee4..f48530d2bb7 100644 --- a/src/backend/replication/logical/tablesync.c +++ b/src/backend/replication/logical/tablesync.c @@ -122,6 +122,7 @@ #include "utils/snapmgr.h" #include "utils/syscache.h" #include "utils/usercontext.h" +#include "utils/acl.h" typedef enum { @@ -1152,6 +1153,7 @@ copy_table(Relation rel) ParseState *pstate; List *options = NIL; bool gencol_published = false; + AclResult aclresult; /* Get the publisher relation info. */ fetch_remote_table_info(get_namespace_name(RelationGetNamespace(rel)), @@ -1165,6 +1167,11 @@ copy_table(Relation rel) relmapentry = logicalrep_rel_open(lrel.remoteid, NoLock); Assert(rel == relmapentry->localrel); + /* Check permission on table. */ + aclresult = pg_class_aclcheck(RelationGetRelid(rel), GetUserId(), ACL_INSERT); + if (aclresult != ACLCHECK_OK) + aclcheck_error(aclresult, get_relkind_objtype(rel->rd_rel->relkind), RelationGetRelationName(rel)); + /* Start copy on the publisher. */ initStringInfo(&cmd); diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c index 2d39a8812f1..6679fc272a9 100644 --- a/src/backend/replication/logical/worker.c +++ b/src/backend/replication/logical/worker.c @@ -185,6 +185,7 @@ #include "utils/snapmgr.h" #include "utils/syscache.h" #include "utils/usercontext.h" +#include "utils/acl.h" #define NAPTIME_PER_CYCLE 1000 /* max sleep time between cycles (1s) */ @@ -2396,6 +2397,7 @@ apply_handle_insert(StringInfo s) TupleTableSlot *remoteslot; MemoryContext oldctx; bool run_as_owner; + AclResult aclresult; /* * Quick return if we are skipping data modification changes or handling @@ -2430,6 +2432,11 @@ apply_handle_insert(StringInfo s) /* Set relation for error callback */ apply_error_callback_arg.rel = rel; + aclresult = pg_class_aclcheck(RelationGetRelid(rel->localrel), GetUserId(), + ACL_INSERT); + if (aclresult != ACLCHECK_OK) + aclcheck_error(aclresult, get_relkind_objtype(rel->localrel->rd_rel->relkind), + RelationGetRelationName(rel->localrel)); /* Initialize the executor state. */ edata = create_edata_for_relation(rel); @@ -2556,6 +2563,7 @@ apply_handle_update(StringInfo s) RTEPermissionInfo *target_perminfo; MemoryContext oldctx; bool run_as_owner; + AclResult aclresult; /* * Quick return if we are skipping data modification changes or handling @@ -2595,6 +2603,14 @@ apply_handle_update(StringInfo s) if (!run_as_owner) SwitchToUntrustedUser(rel->localrel->rd_rel->relowner, &ucxt); + /* MDB check acl on relation */ + aclresult = pg_class_aclcheck(RelationGetRelid(rel->localrel), GetUserId(), + ACL_UPDATE); + if (aclresult != ACLCHECK_OK) + aclcheck_error(aclresult, get_relkind_objtype(rel->localrel->rd_rel->relkind), + RelationGetRelationName(rel->localrel)); + + /* Initialize the executor state. */ edata = create_edata_for_relation(rel); estate = edata->estate; @@ -2761,6 +2777,7 @@ apply_handle_delete(StringInfo s) TupleTableSlot *remoteslot; MemoryContext oldctx; bool run_as_owner; + AclResult aclresult; /* * Quick return if we are skipping data modification changes or handling @@ -2787,7 +2804,6 @@ apply_handle_delete(StringInfo s) /* Set relation for error callback */ apply_error_callback_arg.rel = rel; - /* Check if we can do the delete. */ check_relation_updatable(rel); @@ -2799,6 +2815,12 @@ apply_handle_delete(StringInfo s) if (!run_as_owner) SwitchToUntrustedUser(rel->localrel->rd_rel->relowner, &ucxt); + aclresult = pg_class_aclcheck(RelationGetRelid(rel->localrel), GetUserId(), + ACL_DELETE); + if (aclresult != ACLCHECK_OK) + aclcheck_error(aclresult, get_relkind_objtype(rel->localrel->rd_rel->relkind), + RelationGetRelationName(rel->localrel)); + /* Initialize the executor state. */ edata = create_edata_for_relation(rel); estate = edata->estate; @@ -3249,6 +3271,7 @@ apply_handle_truncate(StringInfo s) List *relids_logged = NIL; ListCell *lc; LOCKMODE lockmode = AccessExclusiveLock; + AclResult aclresult; /* * Quick return if we are skipping data modification changes or handling @@ -3278,6 +3301,12 @@ apply_handle_truncate(StringInfo s) continue; } + aclresult = pg_class_aclcheck(RelationGetRelid(rel->localrel), GetUserId(), + ACL_TRUNCATE); + if (aclresult != ACLCHECK_OK) + aclcheck_error(aclresult, get_relkind_objtype(rel->localrel->rd_rel->relkind), + RelationGetRelationName(rel->localrel)); + remote_rels = lappend(remote_rels, rel); TargetPrivilegesCheck(rel->localrel, ACL_TRUNCATE); rels = lappend(rels, rel->localrel); diff --git a/src/backend/replication/slot.c b/src/backend/replication/slot.c index 4246d0a51e1..521073af3b9 100644 --- a/src/backend/replication/slot.c +++ b/src/backend/replication/slot.c @@ -58,6 +58,7 @@ #include "utils/guc_hooks.h" #include "utils/injection_point.h" #include "utils/varlena.h" +#include "utils/acl.h" /* * Replication slot on-disk data structure. @@ -1553,6 +1554,43 @@ CheckSlotPermissions(void) "REPLICATION"))); } + + +/* + * Check whether the user has privilege to use replication slots. + */ +void +CheckRoleMDBReplSlotPermissions(bool role_has_rolreplication, bool is_member_of_mdb_replication) +{ + /* mdb_replication can do it */ + if (is_member_of_mdb_replication) { + return; + } + + if (!role_has_rolreplication) + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("permission denied to use replication slots"), + errdetail("must be superuser, replication role or mdb_replication to use replication slots"))); +} + + +void +CheckRoleUseMDBReservedName(const char *name, bool role_has_rolreplication) +{ + /* ugly coding for speed (taken from IsReservedName) */ + if (name[0] == 'm' && + name[1] == 'd' && + name[2] == 'b' && + !role_has_rolreplication) + { + ereport(ERROR, + (errcode(ERRCODE_RESERVED_NAME), + errmsg("slot name \"%s\" is reserved", name), + errdetail("Slot names starting with \"mdb\" are reserved."))); + } +} + /* * Reserve WAL for the currently active slot. * diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c index f5b1793c1c5..b6b0ec72d99 100644 --- a/src/backend/replication/slotfuncs.c +++ b/src/backend/replication/slotfuncs.c @@ -23,6 +23,8 @@ #include "utils/builtins.h" #include "utils/guc.h" #include "utils/pg_lsn.h" +#include "utils/resowner.h" +#include "utils/acl.h" /* * Helper function for creating a new physical replication slot with @@ -77,6 +79,7 @@ pg_create_physical_replication_slot(PG_FUNCTION_ARGS) elog(ERROR, "return type must be a row type"); CheckSlotPermissions(); + CheckMDBReservedName(NameStr(*name)); CheckSlotRequirements(); @@ -182,7 +185,8 @@ pg_create_logical_replication_slot(PG_FUNCTION_ARGS) if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) elog(ERROR, "return type must be a row type"); - CheckSlotPermissions(); + CheckMDBReplSlotPermissions(); + CheckMDBReservedName(NameStr(*name)); CheckLogicalDecodingRequirements(); @@ -219,7 +223,9 @@ pg_drop_replication_slot(PG_FUNCTION_ARGS) { Name name = PG_GETARG_NAME(0); - CheckSlotPermissions(); + /* mdb replication allowed */ + CheckMDBReplSlotPermissions(); + CheckMDBReservedName(NameStr(*name)); CheckSlotRequirements(); @@ -521,7 +527,8 @@ pg_replication_slot_advance(PG_FUNCTION_ARGS) Assert(!MyReplicationSlot); - CheckSlotPermissions(); + CheckMDBReplSlotPermissions(); + CheckMDBReservedName(NameStr(*slotname)); if (XLogRecPtrIsInvalid(moveto)) ereport(ERROR, @@ -622,6 +629,8 @@ copy_replication_slot(FunctionCallInfo fcinfo, bool logical_slot) elog(ERROR, "return type must be a row type"); CheckSlotPermissions(); + CheckMDBReservedName(NameStr(*src_name)); + CheckMDBReservedName(NameStr(*dst_name)); if (logical_slot) CheckLogicalDecodingRequirements(); diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 70d90699cc6..3d8efbb69cc 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -118,6 +118,13 @@ WalSnd *MyWalSnd = NULL; /* Global state */ bool am_walsender = false; /* Am I a walsender process? */ + +/* These variables defined in InitPostgres and used in walsender logic for priv +* check without CatCache search. +*/ +bool role_has_rolreplication = false; /* has replication privelege */ +bool member_of_mdb_replication = false; /* member of mdb replication role */ + bool am_cascading_walsender = false; /* Am I cascading WAL to another * standby? */ bool am_db_walsender = false; /* Connected to a database? */ @@ -222,6 +229,20 @@ typedef struct /* The size of our buffer of time samples. */ #define LAG_TRACKER_BUFFER_SIZE 8192 +static void +check_permissions(void) +{ + /* superuser can do it */ + if (superuser()) { + return; + } + /* else should have REPLICATION role option */ + if (!role_has_rolreplication) + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + (errmsg("must be superuser or replication role to use replication slots")))); +} + /* A mechanism for tracking replication lag. */ typedef struct { @@ -1210,6 +1231,7 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd) if (cmd->kind == REPLICATION_KIND_PHYSICAL) { + check_permissions(); ReplicationSlotCreate(cmd->slotname, false, cmd->temporary ? RS_TEMPORARY : RS_PERSISTENT, false, false, false); @@ -1233,6 +1255,8 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd) Assert(cmd->kind == REPLICATION_KIND_LOGICAL); CheckLogicalDecodingRequirements(); + CheckRoleMDBReplSlotPermissions(role_has_rolreplication, member_of_mdb_replication); + CheckRoleUseMDBReservedName(cmd->slotname, role_has_rolreplication); /* * Initially create persistent slot as ephemeral - that allows us to @@ -1395,6 +1419,7 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd) static void DropReplicationSlot(DropReplicationSlotCmd *cmd) { + CheckRoleUseMDBReservedName(cmd->slotname, role_has_rolreplication); ReplicationSlotDrop(cmd->slotname, !cmd->wait); } @@ -1449,6 +1474,8 @@ StartLogicalReplication(StartReplicationCmd *cmd) StringInfoData buf; QueryCompletion qc; + CheckRoleUseMDBReservedName(cmd->slotname, role_has_rolreplication); + /* make sure that our requirements are still fulfilled */ CheckLogicalDecodingRequirements(); @@ -2136,6 +2163,7 @@ exec_replication_command(const char *cmd_string) break; case T_BaseBackupCmd: + check_permissions(); cmdtag = "BASE_BACKUP"; set_ps_display(cmdtag); PreventInTransactionBlock(true, cmdtag); diff --git a/src/backend/utils/init/postinit.c b/src/backend/utils/init/postinit.c index c86ceefda94..35efe314cf3 100644 --- a/src/backend/utils/init/postinit.c +++ b/src/backend/utils/init/postinit.c @@ -718,7 +718,7 @@ InitPostgres(const char *in_dbname, Oid dboid, bool am_superuser; char *fullpath; char dbname[NAMEDATALEN]; - int nfree = 0; + int nfree; elog(DEBUG3, "InitPostgres"); @@ -955,8 +955,16 @@ InitPostgres(const char *in_dbname, Oid dboid, if (am_walsender) { Assert(!bootstrap); - - if (!has_rolreplication(GetUserId())) + /* define this variable for later use in permission checks function. */ + /* we cannot use has_rolreplication directly because catcache search is prohibited + * in no active tx state. + */ + role_has_rolreplication = has_rolreplication(GetUserId()); + member_of_mdb_replication = is_member_of_role(GetUserId(), get_role_oid("mdb_replication", true)); + + /* has_rolreplication returns true in case of superuser_arg(role) */ + /* should have REPLICATION role or be a member of mdb_replication to start walsender */ + if (!role_has_rolreplication && !member_of_mdb_replication) ereport(FATAL, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("permission denied to start WAL sender"), diff --git a/src/include/replication/slot.h b/src/include/replication/slot.h index d9aab934048..79d559752a7 100644 --- a/src/include/replication/slot.h +++ b/src/include/replication/slot.h @@ -9,6 +9,8 @@ #ifndef SLOT_H #define SLOT_H +#include "utils/acl.h" +#include "miscadmin.h" #include "access/xlog.h" #include "access/xlogreader.h" #include "storage/condition_variable.h" @@ -324,4 +326,29 @@ extern bool SlotExistsInSyncStandbySlots(const char *slot_name); extern bool StandbySlotsHaveCaughtup(XLogRecPtr wait_for_lsn, int elevel); extern void WaitForStandbyConfirmation(XLogRecPtr wait_for_lsn); +/* +* Base function for CheckMDBReplSlotPermissions, but does not +* perform cat cache search (for no-transaction state case), +* using pre-defined bool variables. +*/ +extern void CheckRoleMDBReplSlotPermissions(bool role_has_rolreplication, bool is_member_of_mdb_replication); + +/* +* Same as CheckMDBReservedName, but does not +* perform cat cache search (for no-transaction state case), +* using pre-defined bool variables, defined +* in InitPostrges. +*/ +extern void CheckRoleUseMDBReservedName(const char* name, bool role_has_rolreplication); + +inline void CheckMDBReservedName(const char* name) { + CheckRoleUseMDBReservedName(name, has_rolreplication(GetUserId())); +} + +inline void CheckMDBReplSlotPermissions(void) { + Oid role; + role = get_role_oid("mdb_replication", /* missing ok*/ true); + return CheckRoleMDBReplSlotPermissions(has_rolreplication(GetUserId()), is_member_of_role(GetUserId(), role)); +} + #endif /* SLOT_H */ diff --git a/src/include/replication/walsender.h b/src/include/replication/walsender.h index c3e8e191339..e98fe91fc24 100644 --- a/src/include/replication/walsender.h +++ b/src/include/replication/walsender.h @@ -30,6 +30,10 @@ extern PGDLLIMPORT bool am_cascading_walsender; extern PGDLLIMPORT bool am_db_walsender; extern PGDLLIMPORT bool wake_wal_senders; +/* for mdb permission checks */ +extern PGDLLIMPORT bool role_has_rolreplication; +extern PGDLLIMPORT bool member_of_mdb_replication; + /* user-settable parameters */ extern PGDLLIMPORT int max_wal_senders; extern PGDLLIMPORT int wal_sender_timeout; diff --git a/src/test/recovery/t/101_logical_decoding_mdb_replication.pl b/src/test/recovery/t/101_logical_decoding_mdb_replication.pl new file mode 100644 index 00000000000..106e2484db7 --- /dev/null +++ b/src/test/recovery/t/101_logical_decoding_mdb_replication.pl @@ -0,0 +1,159 @@ +# Testing of logical decoding using SQL interface and/or pg_recvlogical +# +# Most logical decoding tests are in contrib/test_decoding. This module +# is for work that doesn't fit well there, like where server restarts +# are required. +use strict; +use warnings; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More tests => 10; +use Config; + + +# Initialize master node +my $node_master = PostgreSQL::Test::Cluster->new('master'); +$node_master->init(allows_streaming => 1); +$node_master->append_conf( + 'postgresql.conf', qq( +wal_level = logical +)); +$node_master->start; +my $backup_name = 'master_backup'; + +$node_master->safe_psql('postgres', + "SET password_encryption='md5'; CREATE ROLE normal LOGIN PASSWORD 'pass';"); +$node_master->safe_psql('postgres', + "GRANT CREATE ON DATABASE postgres TO normal;"); +$node_master->safe_psql('postgres', + "GRANT ALL ON SCHEMA public TO normal;"); +$node_master->safe_psql('postgres', + "CREATE ROLE mdb_replication NOLOGIN;"); +$node_master->safe_psql('postgres', + "GRANT mdb_replication TO normal;"); + +$node_master->safe_psql('postgres', + qq[CREATE TABLE decoding_test(x integer, y text);], extra_params => [ '-U', 'normal' ]); + +$node_master->safe_psql('postgres', + qq[SELECT pg_create_logical_replication_slot('test_slot', 'test_decoding');], + extra_params => [ '-U', 'normal' ]); + +$node_master->safe_psql('postgres', + qq[INSERT INTO decoding_test(x,y) SELECT s, s::text FROM generate_series(1,10) s;], + extra_params => [ '-U', 'normal' ] +); + +# Basic decoding works +my ($result) = $node_master->safe_psql('postgres', + qq[SELECT pg_logical_slot_get_changes('test_slot', NULL, NULL);], + extra_params => [ '-U', 'normal' ]); +is(scalar(my @foobar = split /^/m, $result), + 12, 'Decoding produced 12 rows inc BEGIN/COMMIT'); + +# If we immediately crash the server we might lose the progress we just made +# and replay the same changes again. But a clean shutdown should never repeat +# the same changes when we use the SQL decoding interface. +$node_master->restart; + +# There are no new writes, so the result should be empty. +$result = $node_master->safe_psql('postgres', + qq[SELECT pg_logical_slot_get_changes('test_slot', NULL, NULL);], + extra_params => [ '-U', 'normal' ]); +chomp($result); +is($result, '', 'Decoding after fast restart repeats no rows'); + +# Insert some rows and verify that we get the same results from pg_recvlogical +# and the SQL interface. +$node_master->safe_psql('postgres', + qq[INSERT INTO decoding_test(x,y) SELECT s, s::text FROM generate_series(1,4) s;], + extra_params => [ '-U', 'normal' ] +); + +my $expected = q{BEGIN +table public.decoding_test: INSERT: x[integer]:1 y[text]:'1' +table public.decoding_test: INSERT: x[integer]:2 y[text]:'2' +table public.decoding_test: INSERT: x[integer]:3 y[text]:'3' +table public.decoding_test: INSERT: x[integer]:4 y[text]:'4' +COMMIT}; + +my $stdout_sql = $node_master->safe_psql('postgres', + qq[SELECT data FROM pg_logical_slot_peek_changes('test_slot', NULL, NULL, 'include-xids', '0', 'skip-empty-xacts', '1');], + extra_params => [ '-U', 'normal' ] +); +is($stdout_sql, $expected, 'got expected output from SQL decoding session'); + +my $endpos = $node_master->safe_psql('postgres', + "SELECT lsn FROM pg_logical_slot_peek_changes('test_slot', NULL, NULL) ORDER BY lsn DESC LIMIT 1;", + extra_params => [ '-U', 'normal' ] +); +print "waiting to replay $endpos\n"; + +my $stdout_recv = $node_master->pg_recvlogical_upto( + 'postgres', 'test_slot', $endpos, 180, + 'include-xids' => '0', + 'skip-empty-xacts' => '1'); +chomp($stdout_recv); +is($stdout_recv, $expected, + 'got same expected output from pg_recvlogical decoding session'); + +$node_master->poll_query_until('postgres', + "SELECT EXISTS (SELECT 1 FROM pg_replication_slots WHERE slot_name = 'test_slot' AND active_pid IS NULL)" +) or die "slot never became inactive"; + +$stdout_recv = $node_master->pg_recvlogical_upto( + 'postgres', 'test_slot', $endpos, 180, + 'include-xids' => '0', + 'skip-empty-xacts' => '1'); +chomp($stdout_recv); +is($stdout_recv, '', + 'pg_recvlogical acknowledged changes, nothing pending on slot'); + +$node_master->safe_psql('postgres', 'CREATE DATABASE otherdb'); + +is( $node_master->psql( + 'otherdb', + "SELECT lsn FROM pg_logical_slot_peek_changes('test_slot', NULL, NULL) ORDER BY lsn DESC LIMIT 1;", + extra_params => [ '-U', 'normal' ] + ), + 3, + 'replaying logical slot from another database fails'); + +$node_master->safe_psql('otherdb', + qq[SELECT pg_create_logical_replication_slot('otherdb_slot', 'test_decoding');], + extra_params => [ '-U', 'normal' ] +); + +# make sure you can't drop a slot while active +SKIP: +{ + + # some Windows Perls at least don't like IPC::Run's start/kill_kill regime. + skip "Test fails on Windows perl", 2 if $Config{osname} eq 'MSWin32'; + + my $pg_recvlogical = IPC::Run::start( + [ + 'pg_recvlogical', '-d', $node_master->connstr('otherdb'), + '-S', 'otherdb_slot', '-f', '-', '--start', '-U' , 'normal' + ]); + $node_master->poll_query_until('otherdb', + "SELECT EXISTS (SELECT 1 FROM pg_replication_slots WHERE slot_name = 'otherdb_slot' AND active_pid IS NOT NULL)" + ) or die "slot never became active"; + is($node_master->psql('postgres', 'DROP DATABASE otherdb'), + 3, 'dropping a DB with active logical slots fails'); + $pg_recvlogical->kill_kill; + is($node_master->slot('otherdb_slot')->{'slot_name'}, + undef, 'logical slot still exists'); +} + +$node_master->poll_query_until('otherdb', + "SELECT EXISTS (SELECT 1 FROM pg_replication_slots WHERE slot_name = 'otherdb_slot' AND active_pid IS NULL)", +) or die "slot never became inactive"; + +is($node_master->psql('postgres', 'DROP DATABASE otherdb'), + 0, 'dropping a DB with inactive logical slots succeeds'); +is($node_master->slot('otherdb_slot')->{'slot_name'}, + undef, 'logical slot was actually dropped with DB'); + +# done with the node +$node_master->stop; diff --git a/src/test/regress/expected/mdb_replication.out b/src/test/regress/expected/mdb_replication.out new file mode 100644 index 00000000000..2c8f7bac409 --- /dev/null +++ b/src/test/regress/expected/mdb_replication.out @@ -0,0 +1,35 @@ +CREATE ROLE regress_mdb_repl_no_priv LOGIN NOSUPERUSER; +CREATE ROLE regress_mdb_repl_no_priv2 LOGIN NOSUPERUSER; +CREATE ROLE regress_mdb_repl LOGIN NOSUPERUSER; +CREATE ROLE regress_mdb_repl_su LOGIN SUPERUSER; +CREATE ROLE regress_mdb_repl_pgrad LOGIN NOSUPERUSER; +GRANT pg_read_all_data TO regress_mdb_repl_pgrad; +GRANT mdb_admin to regress_mdb_repl; +GRANT CREATE ON DATABASE REGRESSION TO regress_mdb_repl; +GRANT CREATE ON DATABASE REGRESSION TO regress_mdb_repl_no_priv; +-- ok - member of mdb_admin +SET SESSION AUTHORIZATION regress_mdb_repl; +CREATE SUBSCRIPTION regress_mdbsub CONNECTION 'dbname=doesnotexist password=regress_fakepassword' PUBLICATION foo WITH (slot_name = NONE, connect = false); +WARNING: subscription was created, but is not connected +HINT: To initiate replication, you must manually create the replication slot, enable the subscription, and refresh the subscription. +-- ok - we are allowed to change ownership under mdb_admin +ALTER SUBSCRIPTION regress_mdbsub OWNER TO regress_mdb_repl_no_priv2; +-- should fail - we are not allowed to change ownership to priviledged role +ALTER SUBSCRIPTION regress_mdbsub OWNER TO regress_mdb_repl_su; +ERROR: must be owner of subscription regress_mdbsub +ALTER SUBSCRIPTION regress_mdbsub OWNER TO regress_mdb_repl_pgrad; +ERROR: must be owner of subscription regress_mdbsub +RESET SESSION AUTHORIZATION; +DROP SUBSCRIPTION regress_mdbsub; +-- should fail - not member of pg_subcription_users or mdb_admin +SET SESSION AUTHORIZATION regress_mdb_repl_no_priv; +CREATE SUBSCRIPTION regress_mdbsub_no_priv CONNECTION 'dbname=doesnotexist password=regress_fakepassword' PUBLICATION foo WITH (slot_name = NONE, connect = false); +ERROR: permission denied to create subscription +DETAIL: Only roles with privileges of the "pg_create_subscription" role may create subscriptions. +-- reset to su, cleanup +RESET SESSION AUTHORIZATION; +REVOKE ALL ON DATABASE REGRESSION FROM regress_mdb_repl; +REVOKE ALL ON DATABASE REGRESSION FROM regress_mdb_repl_no_priv; +DROP ROLE regress_mdb_repl; +DROP ROLE regress_mdb_repl_no_priv; +DROP ROLE regress_mdb_repl_no_priv2; diff --git a/src/test/regress/expected/test_setup.out b/src/test/regress/expected/test_setup.out index b6ace34542b..da37e9d1304 100644 --- a/src/test/regress/expected/test_setup.out +++ b/src/test/regress/expected/test_setup.out @@ -236,3 +236,6 @@ create function fipshash(text) strict immutable parallel safe leakproof return substr(encode(sha256($1::bytea), 'hex'), 1, 32); CREATE ROLE mdb_admin; +CREATE ROLE mdb_superuser; +CREATE ROLE mdb_replication; +GRANT pg_create_subscription TO mdb_admin; diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule index afb347ec600..59abe787bde 100644 --- a/src/test/regress/parallel_schedule +++ b/src/test/regress/parallel_schedule @@ -15,6 +15,10 @@ test: test_setup test: mdb_admin +test: mdb_superuser + +test: mdb_replication + # ---------- # The first group of parallel tests # ---------- diff --git a/src/test/regress/sql/mdb_replication.sql b/src/test/regress/sql/mdb_replication.sql new file mode 100644 index 00000000000..31bcb460879 --- /dev/null +++ b/src/test/regress/sql/mdb_replication.sql @@ -0,0 +1,41 @@ +CREATE ROLE regress_mdb_repl_no_priv LOGIN NOSUPERUSER; +CREATE ROLE regress_mdb_repl_no_priv2 LOGIN NOSUPERUSER; +CREATE ROLE regress_mdb_repl LOGIN NOSUPERUSER; +CREATE ROLE regress_mdb_repl_su LOGIN SUPERUSER; +CREATE ROLE regress_mdb_repl_pgrad LOGIN NOSUPERUSER; + +GRANT pg_read_all_data TO regress_mdb_repl_pgrad; + +GRANT mdb_admin to regress_mdb_repl; + +GRANT CREATE ON DATABASE REGRESSION TO regress_mdb_repl; + +GRANT CREATE ON DATABASE REGRESSION TO regress_mdb_repl_no_priv; + +-- ok - member of mdb_admin +SET SESSION AUTHORIZATION regress_mdb_repl; +CREATE SUBSCRIPTION regress_mdbsub CONNECTION 'dbname=doesnotexist password=regress_fakepassword' PUBLICATION foo WITH (slot_name = NONE, connect = false); + +-- ok - we are allowed to change ownership under mdb_admin +ALTER SUBSCRIPTION regress_mdbsub OWNER TO regress_mdb_repl_no_priv2; + +-- should fail - we are not allowed to change ownership to priviledged role +ALTER SUBSCRIPTION regress_mdbsub OWNER TO regress_mdb_repl_su; +ALTER SUBSCRIPTION regress_mdbsub OWNER TO regress_mdb_repl_pgrad; + +RESET SESSION AUTHORIZATION; +DROP SUBSCRIPTION regress_mdbsub; + +-- should fail - not member of pg_subcription_users or mdb_admin +SET SESSION AUTHORIZATION regress_mdb_repl_no_priv; +CREATE SUBSCRIPTION regress_mdbsub_no_priv CONNECTION 'dbname=doesnotexist password=regress_fakepassword' PUBLICATION foo WITH (slot_name = NONE, connect = false); +-- reset to su, cleanup + +RESET SESSION AUTHORIZATION; + +REVOKE ALL ON DATABASE REGRESSION FROM regress_mdb_repl; +REVOKE ALL ON DATABASE REGRESSION FROM regress_mdb_repl_no_priv; + +DROP ROLE regress_mdb_repl; +DROP ROLE regress_mdb_repl_no_priv; +DROP ROLE regress_mdb_repl_no_priv2; \ No newline at end of file diff --git a/src/test/regress/sql/test_setup.sql b/src/test/regress/sql/test_setup.sql index ed46cf1f2eb..8fd2b005bc6 100644 --- a/src/test/regress/sql/test_setup.sql +++ b/src/test/regress/sql/test_setup.sql @@ -291,3 +291,7 @@ create function fipshash(text) return substr(encode(sha256($1::bytea), 'hex'), 1, 32); CREATE ROLE mdb_admin; +CREATE ROLE mdb_superuser; +CREATE ROLE mdb_replication; + +GRANT pg_create_subscription TO mdb_admin; diff --git a/src/test/subscription/t/101_mdb_rep_changes_nonsuperuser.pl b/src/test/subscription/t/101_mdb_rep_changes_nonsuperuser.pl new file mode 100644 index 00000000000..0d3ae411ddf --- /dev/null +++ b/src/test/subscription/t/101_mdb_rep_changes_nonsuperuser.pl @@ -0,0 +1,336 @@ +# Basic logical replication test +use strict; +use warnings; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +# Initialize publisher node +my $node_publisher = PostgreSQL::Test::Cluster->new('publisher'); +$node_publisher->init(allows_streaming => 'logical'); +$node_publisher->start; + +# Create subscriber node +my $node_subscriber = PostgreSQL::Test::Cluster->new('subscriber'); +$node_subscriber->init(allows_streaming => 'logical'); +$node_subscriber->start; + +# Prepare publisher +$node_publisher->safe_psql('postgres', + "CREATE ROLE normal_mdb_repl LOGIN PASSWORD 'regress_lolkekpassword'"); +$node_publisher->safe_psql('postgres', + "CREATE ROLE mdb_replication"); +$node_publisher->safe_psql('postgres', + "GRANT mdb_replication TO normal_mdb_repl"); +$node_publisher->safe_psql('postgres', + "GRANT CREATE ON DATABASE postgres TO normal_mdb_repl"); +$node_publisher->safe_psql('postgres', + "GRANT ALL ON SCHEMA public TO normal_mdb_repl;"); + + +# Delete pg_hba.conf from the given node, add a new entry to it +# and then execute a reload to refresh it. +sub reset_pg_hba +{ + my $node = shift; + my $hba_method = shift; + + unlink($node->data_dir . '/pg_hba.conf'); + $node->append_conf('pg_hba.conf', "local all normal_mdb_repl $hba_method"); + $node->append_conf('pg_hba.conf', "local all all trust"); + $node->reload; + return; +} + +# Prepare subscriber +$node_subscriber->safe_psql('postgres', + "CREATE ROLE normal_mdb_admin LOGIN"); +$node_subscriber->safe_psql('postgres', + "CREATE ROLE mdb_admin"); +$node_subscriber->safe_psql('postgres', + "GRANT pg_create_subscription TO mdb_admin"); +$node_subscriber->safe_psql('postgres', + "GRANT mdb_admin TO normal_mdb_admin"); +$node_subscriber->safe_psql('postgres', + "GRANT CREATE ON DATABASE postgres TO normal_mdb_admin"); +$node_subscriber->safe_psql('postgres', + "GRANT ALL ON SCHEMA public TO normal_mdb_admin;"); + +# Create some preexisting content on publisher +$node_publisher->safe_psql('postgres', + "CREATE TABLE tab_notrep AS SELECT generate_series(1,10) AS a", + extra_params => [ '-U', 'normal_mdb_repl' ]); +$node_publisher->safe_psql('postgres', + "CREATE TABLE tab_ins AS SELECT generate_series(1,1002) AS a", + extra_params => [ '-U', 'normal_mdb_repl' ]); +$node_publisher->safe_psql('postgres', + "CREATE TABLE tab_full AS SELECT generate_series(1,10) AS a", + extra_params => [ '-U', 'normal_mdb_repl' ]); +$node_publisher->safe_psql('postgres', "CREATE TABLE tab_full2 (x text)", + extra_params => [ '-U', 'normal_mdb_repl' ]); +$node_publisher->safe_psql('postgres', + "INSERT INTO tab_full2 VALUES ('a'), ('b'), ('b')", + extra_params => [ '-U', 'normal_mdb_repl' ]); +$node_publisher->safe_psql('postgres', + "CREATE TABLE tab_rep (a int primary key)", + extra_params => [ '-U', 'normal_mdb_repl' ]); +$node_publisher->safe_psql('postgres', + "CREATE TABLE tab_mixed (a int primary key, b text)", + extra_params => [ '-U', 'normal_mdb_repl' ]); +$node_publisher->safe_psql('postgres', + "INSERT INTO tab_mixed (a, b) VALUES (1, 'foo')", + extra_params => [ '-U', 'normal_mdb_repl' ]); +$node_publisher->safe_psql('postgres', + "CREATE TABLE tab_include (a int, b text, CONSTRAINT covering PRIMARY KEY(a) INCLUDE(b))", + extra_params => [ '-U', 'normal_mdb_repl' ] +); + + +# Setup structure on subscriber +$node_subscriber->safe_psql('postgres', "CREATE TABLE tab_notrep (a int)", extra_params => [ '-U', 'normal_mdb_admin' ]); +$node_subscriber->safe_psql('postgres', "CREATE TABLE tab_ins (a int)", extra_params => [ '-U', 'normal_mdb_admin' ]); +$node_subscriber->safe_psql('postgres', "CREATE TABLE tab_full (a int)", extra_params => [ '-U', 'normal_mdb_admin' ]); +$node_subscriber->safe_psql('postgres', "CREATE TABLE tab_full2 (x text)", extra_params => [ '-U', 'normal_mdb_admin' ]); +$node_subscriber->safe_psql('postgres', + "CREATE TABLE tab_rep (a int primary key)", + extra_params => [ '-U', 'normal_mdb_admin' ]); + +# different column count and order than on publisher +$node_subscriber->safe_psql('postgres', + "CREATE TABLE tab_mixed (c text, b text, a int primary key)", + extra_params => [ '-U', 'normal_mdb_admin' ]); + +# replication of the table with included index +$node_subscriber->safe_psql('postgres', + "CREATE TABLE tab_include (a int, b text, CONSTRAINT covering PRIMARY KEY(a) INCLUDE(b))", + extra_params => [ '-U', 'normal_mdb_admin' ] +); + +# Setup logical replication +my $publisher_connstr = $node_publisher->connstr . ' dbname=postgres user=normal_mdb_repl'; +$node_publisher->safe_psql('postgres', "CREATE PUBLICATION tap_pub", extra_params => [ '-U', 'normal_mdb_repl' ]); +$node_publisher->safe_psql('postgres', + "CREATE PUBLICATION tap_pub_ins_only WITH (publish = insert)", + extra_params => [ '-U', 'normal_mdb_repl' ]); +$node_publisher->safe_psql('postgres', + "ALTER PUBLICATION tap_pub ADD TABLE tab_rep, tab_full, tab_full2, tab_mixed, tab_include", + extra_params => [ '-U', 'normal_mdb_repl' ] +); +$node_publisher->safe_psql('postgres', + "ALTER PUBLICATION tap_pub_ins_only ADD TABLE tab_ins", + extra_params => [ '-U', 'normal_mdb_repl' ]); + + +# require password for all connection on publisher +reset_pg_hba($node_publisher, 'scram-sha-256'); + +$ENV{"PGPASSWORD"} = 'regress_lolkekpassword'; + +my $appname = 'tap_sub'; +$node_subscriber->safe_psql('postgres', + "CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr password=regress_lolkekpassword application_name=$appname' PUBLICATION tap_pub, tap_pub_ins_only", + extra_params => [ '-U', 'normal_mdb_admin' ] +); + +$node_publisher->wait_for_catchup($appname); + +# Also wait for initial table sync to finish +my $synced_query = + "SELECT count(1) = 0 FROM pg_subscription_rel WHERE srsubstate NOT IN ('r', 's');"; +$node_subscriber->poll_query_until('postgres', $synced_query) + or die "Timed out while waiting for subscriber to synchronize data"; + +my $result = + $node_subscriber->safe_psql('postgres', "SELECT count(*) FROM tab_notrep"); +is($result, qq(0), 'check non-replicated table is empty on subscriber'); + +$result = + $node_subscriber->safe_psql('postgres', "SELECT count(*) FROM tab_ins"); +is($result, qq(1002), 'check initial data was copied to subscriber'); + +$node_publisher->safe_psql('postgres', + "INSERT INTO tab_ins SELECT generate_series(1,50)"); +$node_publisher->safe_psql('postgres', "DELETE FROM tab_ins WHERE a > 20"); +$node_publisher->safe_psql('postgres', "UPDATE tab_ins SET a = -a"); + +$node_publisher->safe_psql('postgres', + "INSERT INTO tab_rep SELECT generate_series(1,50)"); +$node_publisher->safe_psql('postgres', "DELETE FROM tab_rep WHERE a > 20"); +$node_publisher->safe_psql('postgres', "UPDATE tab_rep SET a = -a"); + +$node_publisher->safe_psql('postgres', + "INSERT INTO tab_mixed VALUES (2, 'bar')"); + +$node_publisher->safe_psql('postgres', + "INSERT INTO tab_include SELECT generate_series(1,50)"); +$node_publisher->safe_psql('postgres', + "DELETE FROM tab_include WHERE a > 20"); +$node_publisher->safe_psql('postgres', "UPDATE tab_include SET a = -a"); + +$node_publisher->wait_for_catchup($appname); + +$result = $node_subscriber->safe_psql('postgres', + "SELECT count(*), min(a), max(a) FROM tab_ins"); +is($result, qq(1052|1|1002), 'check replicated inserts on subscriber'); + +$result = $node_subscriber->safe_psql('postgres', + "SELECT count(*), min(a), max(a) FROM tab_rep"); +is($result, qq(20|-20|-1), 'check replicated changes on subscriber'); + +$result = + $node_subscriber->safe_psql('postgres', "SELECT c, b, a FROM tab_mixed"); +is( $result, qq(|foo|1 +|bar|2), 'check replicated changes with different column order'); + +$result = $node_subscriber->safe_psql('postgres', + "SELECT count(*), min(a), max(a) FROM tab_include"); +is($result, qq(20|-20|-1), + 'check replicated changes with primary key index with included columns'); + +# insert some duplicate rows +$node_publisher->safe_psql('postgres', + "INSERT INTO tab_full SELECT generate_series(1,10)"); + +# add REPLICA IDENTITY FULL so we can update +$node_publisher->safe_psql('postgres', + "ALTER TABLE tab_full REPLICA IDENTITY FULL"); +$node_subscriber->safe_psql('postgres', + "ALTER TABLE tab_full REPLICA IDENTITY FULL"); +$node_publisher->safe_psql('postgres', + "ALTER TABLE tab_full2 REPLICA IDENTITY FULL"); +$node_subscriber->safe_psql('postgres', + "ALTER TABLE tab_full2 REPLICA IDENTITY FULL"); +$node_publisher->safe_psql('postgres', + "ALTER TABLE tab_ins REPLICA IDENTITY FULL"); +$node_subscriber->safe_psql('postgres', + "ALTER TABLE tab_ins REPLICA IDENTITY FULL"); + +# and do the updates +$node_publisher->safe_psql('postgres', "UPDATE tab_full SET a = a * a"); +$node_publisher->safe_psql('postgres', + "UPDATE tab_full2 SET x = 'bb' WHERE x = 'b'"); + +$node_publisher->wait_for_catchup($appname); + +$result = $node_subscriber->safe_psql('postgres', + "SELECT count(*), min(a), max(a) FROM tab_full"); +is($result, qq(20|1|100), + 'update works with REPLICA IDENTITY FULL and duplicate tuples'); + +$result = $node_subscriber->safe_psql('postgres', + "SELECT x FROM tab_full2 ORDER BY 1"); +is( $result, qq(a +bb +bb), + 'update works with REPLICA IDENTITY FULL and text datums'); + +# check that change of connection string and/or publication list causes +# restart of subscription workers. Not all of these are registered as tests +# as we need to poll for a change but the test suite will fail none the less +# when something goes wrong. +my $oldpid = $node_publisher->safe_psql('postgres', + "SELECT pid FROM pg_stat_replication WHERE application_name = '$appname';" +); +$node_subscriber->safe_psql('postgres', + "ALTER SUBSCRIPTION tap_sub CONNECTION 'application_name=$appname $publisher_connstr password=regress_lolkekpassword '", + extra_params => [ '-U', 'normal_mdb_admin' ] +); +$node_publisher->poll_query_until('postgres', + "SELECT pid != $oldpid FROM pg_stat_replication WHERE application_name = '$appname';" +) or die "Timed out while waiting for apply to restart"; + +$oldpid = $node_publisher->safe_psql('postgres', + "SELECT pid FROM pg_stat_replication WHERE application_name = '$appname';" +); +$node_subscriber->safe_psql('postgres', + "ALTER SUBSCRIPTION tap_sub SET PUBLICATION tap_pub_ins_only WITH (copy_data = false)", + extra_params => [ '-U', 'normal_mdb_admin' ] +); +$node_publisher->poll_query_until('postgres', + "SELECT pid != $oldpid FROM pg_stat_replication WHERE application_name = '$appname';" +) or die "Timed out while waiting for apply to restart"; + +$node_publisher->safe_psql('postgres', + "INSERT INTO tab_ins SELECT generate_series(1001,1100)"); +$node_publisher->safe_psql('postgres', "DELETE FROM tab_rep"); + +# Restart the publisher and check the state of the subscriber which +# should be in a streaming state after catching up. +$node_publisher->stop('fast'); +$node_publisher->start; + +$node_publisher->wait_for_catchup($appname); + +$result = $node_subscriber->safe_psql('postgres', + "SELECT count(*), min(a), max(a) FROM tab_ins"); +is($result, qq(1152|1|1100), + 'check replicated inserts after subscription publication change'); + +$result = $node_subscriber->safe_psql('postgres', + "SELECT count(*), min(a), max(a) FROM tab_rep"); +is($result, qq(20|-20|-1), + 'check changes skipped after subscription publication change'); + +# check alter publication (relcache invalidation etc) +$node_publisher->safe_psql('postgres', + "ALTER PUBLICATION tap_pub_ins_only SET (publish = 'insert, delete')"); +$node_publisher->safe_psql('postgres', + "ALTER PUBLICATION tap_pub_ins_only ADD TABLE tab_full"); +$node_publisher->safe_psql('postgres', "DELETE FROM tab_ins WHERE a > 0"); +$node_subscriber->safe_psql('postgres', + "ALTER SUBSCRIPTION tap_sub REFRESH PUBLICATION WITH (copy_data = false)", + extra_params => [ '-U', 'normal_mdb_admin' ] +); +$node_publisher->safe_psql('postgres', "INSERT INTO tab_full VALUES(0)"); + +$node_publisher->wait_for_catchup($appname); + +# note that data are different on provider and subscriber +$result = $node_subscriber->safe_psql('postgres', + "SELECT count(*), min(a), max(a) FROM tab_ins"); +is($result, qq(1052|1|1002), + 'check replicated deletes after alter publication'); + +$result = $node_subscriber->safe_psql('postgres', + "SELECT count(*), min(a), max(a) FROM tab_full"); +is($result, qq(21|0|100), 'check replicated insert after alter publication'); + +# check restart on rename +$oldpid = $node_publisher->safe_psql('postgres', + "SELECT pid FROM pg_stat_replication WHERE application_name = '$appname';" +); +$node_subscriber->safe_psql('postgres', + "ALTER SUBSCRIPTION tap_sub RENAME TO tap_sub_renamed", + extra_params => [ '-U', 'normal_mdb_admin' ]); +$node_publisher->poll_query_until('postgres', + "SELECT pid != $oldpid FROM pg_stat_replication WHERE application_name = '$appname';" +) or die "Timed out while waiting for apply to restart"; + +# check all the cleanup +$node_subscriber->safe_psql('postgres', "DROP SUBSCRIPTION tap_sub_renamed", extra_params => [ '-U', 'normal_mdb_admin' ]); + +$result = $node_subscriber->safe_psql('postgres', + "SELECT count(*) FROM pg_subscription"); +is($result, qq(0), 'check subscription was dropped on subscriber'); + +$result = $node_publisher->safe_psql('postgres', + "SELECT count(*) FROM pg_replication_slots"); +is($result, qq(0), 'check replication slot was dropped on publisher'); + +$result = $node_subscriber->safe_psql('postgres', + "SELECT count(*) FROM pg_subscription_rel"); +is($result, qq(0), + 'check subscription relation status was dropped on subscriber'); + +$result = $node_publisher->safe_psql('postgres', + "SELECT count(*) FROM pg_replication_slots"); +is($result, qq(0), 'check replication slot was dropped on publisher'); + +$result = $node_subscriber->safe_psql('postgres', + "SELECT count(*) FROM pg_replication_origin"); +is($result, qq(0), 'check replication origin was dropped on subscriber'); + +$node_subscriber->stop('fast'); +$node_publisher->stop('fast'); + +done_testing(); diff --git a/src/test/subscription/t/102_mdb_check_permissions.pl b/src/test/subscription/t/102_mdb_check_permissions.pl new file mode 100644 index 00000000000..1efd1cccf6e --- /dev/null +++ b/src/test/subscription/t/102_mdb_check_permissions.pl @@ -0,0 +1,202 @@ + +# Copyright (c) 2021-2023, PostgreSQL Global Development Group + +# Test that logical replication respects permissions +use strict; +use warnings; +use PostgreSQL::Test::Cluster; +use Test::More; + +my ($node_publisher, $node_subscriber, $publisher_connstr, $result, $offset); +$offset = 0; + +sub publish_insert +{ + my ($tbl, $new_i) = @_; + $node_publisher->safe_psql( + 'postgres', qq( + SET SESSION AUTHORIZATION regress_user1; + INSERT INTO $tbl (i) VALUES ($new_i); + )); +} + +sub publish_update +{ + my ($tbl, $old_i, $new_i) = @_; + $node_publisher->safe_psql( + 'postgres', qq( + SET SESSION AUTHORIZATION regress_user1; + UPDATE $tbl SET i = $new_i WHERE i = $old_i; + )); +} + +sub publish_delete +{ + my ($tbl, $old_i) = @_; + $node_publisher->safe_psql( + 'postgres', qq( + SET SESSION AUTHORIZATION regress_user1; + DELETE FROM $tbl WHERE i = $old_i; + )); +} + + +sub publish_truncate +{ + my ($tbl) = @_; + $node_publisher->safe_psql( + 'postgres', qq( + SET SESSION AUTHORIZATION regress_user1; + TRUNCATE $tbl; + )); +} + +sub expect_replication +{ + my ($tbl, $cnt, $min, $max, $testname) = @_; + $node_publisher->wait_for_catchup('app_test_mdb_admin_sub'); + $result = $node_subscriber->safe_psql( + 'postgres', qq( + SELECT COUNT(i), MIN(i), MAX(i) FROM $tbl)); + is($result, "$cnt|$min|$max", $testname); +} + +sub expect_failure +{ + my ($tbl, $cnt, $min, $max, $re, $testname) = @_; + $offset = $node_subscriber->wait_for_log($re, $offset); + $result = $node_subscriber->safe_psql( + 'postgres', qq( + SELECT COUNT(i), MIN(i), MAX(i) FROM $tbl)); + is($result, "$cnt|$min|$max", $testname); +} + +# Create publisher and subscriber nodes with schemas owned and published by +# "regress_alice" but subscribed and replicated by different role +# "regress_admin". For partitioned tables, layout the partitions differently +# on the publisher than on the subscriber. +# +$node_publisher = PostgreSQL::Test::Cluster->new('publisher'); +$node_subscriber = PostgreSQL::Test::Cluster->new('subscriber'); +$node_publisher->init(allows_streaming => 'logical'); +$node_subscriber->init; +$node_publisher->start; +$node_subscriber->start; +$publisher_connstr = $node_publisher->connstr . ' dbname=postgres'; + +for my $node ($node_publisher, $node_subscriber) +{ + $node->safe_psql( + 'postgres', qq( + CREATE ROLE mdb_admin; + GRANT pg_create_subscription TO mdb_admin; + CREATE ROLE mdb_replication; + + CREATE ROLE regress_user1 NOSUPERUSER LOGIN PASSWORD 'regress_lolkekpassword'; + CREATE ROLE regress_user2 NOSUPERUSER LOGIN PASSWORD 'regress_lolkekpassword'; + + GRANT CREATE ON DATABASE postgres TO regress_user1; + GRANT CREATE ON DATABASE postgres TO regress_user2; + SET SESSION AUTHORIZATION regress_user1; + + CREATE SCHEMA sh; + CREATE TABLE sh.tt (i INTEGER); + + ALTER TABLE sh.tt REPLICA IDENTITY FULL; + + GRANT USAGE ON SCHEMA sh TO regress_user2; + GRANT INSERT, SELECT, UPDATE, DELETE, TRUNCATE ON TABLE sh.tt TO regress_user2; + )); +} + +$node_publisher->safe_psql( + 'postgres', qq( +GRANT mdb_replication TO regress_user2; +SET SESSION AUTHORIZATION regress_user1; + +CREATE PUBLICATION tap_pub + FOR TABLE sh.tt; +)); + + +# Delete pg_hba.conf from the given node, add a new entry to it +# and then execute a reload to refresh it. +sub reset_pg_hba +{ + my $node = shift; + my $hba_method = shift; + + unlink($node->data_dir . '/pg_hba.conf'); + $node->append_conf('pg_hba.conf', "local all regress_user2 $hba_method"); + $node->append_conf('pg_hba.conf', "local all all trust"); + $node->reload; + return; +} + +# require password for all connection on publisher +reset_pg_hba($node_publisher, 'scram-sha-256'); + +$ENV{"PGPASSWORD"} = 'regress_lolkekpassword'; + +$node_subscriber->safe_psql( + 'postgres', qq( +GRANT mdb_admin TO regress_user2; +SET SESSION AUTHORIZATION regress_user2; + +CREATE SUBSCRIPTION test_mdb_admin_sub CONNECTION '$publisher_connstr application_name=app_test_mdb_admin_sub user=regress_user2 password=regress_lolkekpassword' PUBLICATION tap_pub WITH(run_as_owner=true); +)); + +# Wait for initial sync to finish +$node_subscriber->wait_for_subscription_sync($node_publisher, 'app_test_mdb_admin_sub'); + +# Verify that "regress_admin" can replicate into the tables +# +publish_insert("sh.tt", 2); +publish_insert("sh.tt", 8); +publish_insert("sh.tt", 16); +expect_replication("sh.tt", 3, 2, 16, + "mdb_admin/mdb_replication role replicates into sh.tt"); + +# check that no replication possible if permission revoked +$node_subscriber->safe_psql( + 'postgres', qq( + REVOKE INSERT ON TABLE sh.tt FROM regress_user2 +)); + + + +publish_insert("sh.tt", 47); + +expect_failure( + "sh.tt", + 3, + 2, + 16, + qr/ERROR: permission denied for table tt/, + "role without INSERT grant for table fails to replicate insert"); + + +# check that no replication possible if permission revoked +$node_subscriber->safe_psql( + 'postgres', qq( + GRANT INSERT ON TABLE sh.tt TO regress_user2 +)); + + +expect_replication("sh.tt", 4, 2, 47, + "mdb_admin/mdb_replication role replicates after grants again"); + +publish_update("sh.tt", 2 => 7); +publish_delete("sh.tt", 16); + +expect_replication("sh.tt", 3, 7, 47, + "mdb_admin/mdb_replication role replicates update/delete into sh.tt"); + +publish_truncate("sh.tt"); + +publish_insert("sh.tt", 47); + +expect_replication("sh.tt", 1, 47, 47, + "mdb_admin/mdb_replication role replicates truncate sh.tt"); + +done_testing(); From 0e6583bb8240a1f814528a5bc026c32c796aff45 Mon Sep 17 00:00:00 2001 From: Kirill Reshke Date: Wed, 11 Dec 2024 20:14:55 +0300 Subject: [PATCH 10/45] Change storage class for mdb_replication utilities --- src/include/replication/slot.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/include/replication/slot.h b/src/include/replication/slot.h index 79d559752a7..9a95c5fd2d7 100644 --- a/src/include/replication/slot.h +++ b/src/include/replication/slot.h @@ -341,11 +341,11 @@ extern void CheckRoleMDBReplSlotPermissions(bool role_has_rolreplication, bool i */ extern void CheckRoleUseMDBReservedName(const char* name, bool role_has_rolreplication); -inline void CheckMDBReservedName(const char* name) { +static inline void CheckMDBReservedName(const char* name) { CheckRoleUseMDBReservedName(name, has_rolreplication(GetUserId())); } -inline void CheckMDBReplSlotPermissions(void) { +static inline void CheckMDBReplSlotPermissions(void) { Oid role; role = get_role_oid("mdb_replication", /* missing ok*/ true); return CheckRoleMDBReplSlotPermissions(has_rolreplication(GetUserId()), is_member_of_role(GetUserId(), role)); From 1b7ee622814d105ae994f184d1e4cebd84894f1d Mon Sep 17 00:00:00 2001 From: reshke kirill Date: Fri, 27 Jun 2025 14:47:56 +0000 Subject: [PATCH 11/45] Disable security test --- contrib/seg/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/seg/Makefile b/contrib/seg/Makefile index b408f4049cb..ec7d4af7dc3 100644 --- a/contrib/seg/Makefile +++ b/contrib/seg/Makefile @@ -14,7 +14,7 @@ PGFILEDESC = "seg - line segment data type" HEADERS = segdata.h -REGRESS = security seg partition +REGRESS = seg partition EXTRA_CLEAN = segparse.h segparse.c segscan.c From 6a4ed0ce3568fcc830e3d02dbac4367e0a5809e3 Mon Sep 17 00:00:00 2001 From: reshke Date: Sun, 29 Jun 2025 00:52:14 +0500 Subject: [PATCH 12/45] Disable tests with DEB_BUILD_OPTIONS --- docker/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index f887448fe50..2fc4a1e6874 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -10,7 +10,7 @@ cd /home/build-user sudo ./docker/tzdata.sh cat debian/changelog -#export DEB_BUILD_OPTIONS="nocheck" +export DEB_BUILD_OPTIONS="nocheck" sudo dpkg-checkbuilddeps From a0fa7ec22927242affa83f6871a525c31f4cba76 Mon Sep 17 00:00:00 2001 From: reshke kirill Date: Tue, 21 Feb 2023 19:34:14 +0000 Subject: [PATCH 13/45] Role mdb_superuser: feature and regress testsing This commit introduces new mdb internal role mdb_superuser. Role is capaple of: GRANT/REVOKE any set of priviledges to/from any object in database. Has power of pg_database_owner in any database, including: DROP any object in database (except system catalog and stuff) Role is NOT capaple of: Create database, role, extension or alter other roles with such priviledges. Transfer ownership to /pass has_priv of roles: PG_READ_ALL_DATA PG_WRITE_ALL_DATA PG_EXECUTE_SERVER_PROGRAM PG_READ_SERVER_FILES PG_WRITE_SERVER_FILES Fix configure.ac USE_MDBLOCALES option handling Apply autoreconf stuff Set missing ok parameter ito true while acquiring mdb_superuser oid In regress tests, nobody creates mdb_superuser role, so missing ok is fine Fix spelling Applied suggestion Allow mdb_superuser to have power of pg_database_owner Allow mdb_superuser to alter objects and grant ACl to objects, owned by pg_database_owner. Also, when acl check, allow mdb_superuser use pg_database_owner role power to pass check regression test fixes --- src/backend/commands/functioncmds.c | 4 +- src/backend/utils/adt/acl.c | 34 ++++- src/include/pg_config.h.in | 6 + src/include/utils/acl.h | 1 + src/test/regress/expected/mdb_superuser.out | 108 +++++++++++++++ src/test/regress/sql/mdb_superuser.sql | 138 ++++++++++++++++++++ 6 files changed, 288 insertions(+), 3 deletions(-) create mode 100644 src/test/regress/expected/mdb_superuser.out create mode 100644 src/test/regress/sql/mdb_superuser.sql diff --git a/src/backend/commands/functioncmds.c b/src/backend/commands/functioncmds.c index faf4ec08774..0b58e6bc2c7 100644 --- a/src/backend/commands/functioncmds.c +++ b/src/backend/commands/functioncmds.c @@ -1146,7 +1146,7 @@ CreateFunction(ParseState *pstate, CreateFunctionStmt *stmt) */ if (isLeakProof && !superuser()) { - Oid role = get_role_oid("mdb_admin", true); + Oid role = get_role_oid("mdb_admin", true /*if nodoby created mdb_admin role in this database*/); if (!is_member_of_role(GetUserId(), role)) ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), @@ -1437,7 +1437,7 @@ AlterFunction(ParseState *pstate, AlterFunctionStmt *stmt) procForm->proleakproof = boolVal(leakproof_item->arg); if (procForm->proleakproof && !superuser()) { - Oid role = get_role_oid("mdb_admin", true); + Oid role = get_role_oid("mdb_admin", true /*if nodoby created mdb_admin role in this database*/); if (!is_member_of_role(GetUserId(), role)) ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), diff --git a/src/backend/utils/adt/acl.c b/src/backend/utils/adt/acl.c index bb759d0c2ef..f14086ad25a 100644 --- a/src/backend/utils/adt/acl.c +++ b/src/backend/utils/adt/acl.c @@ -5310,7 +5310,7 @@ has_privs_of_role_strict(Oid member, Oid role) /* * Check that role is either one of "dangerous" system role -* or has "strict" (not through mdb_admin) +* or has "strict" (not through mdb_admin or mdb_superuser) * privs of this role */ @@ -5338,6 +5338,8 @@ has_privs_of_unwanted_system_role(Oid role) { bool has_privs_of_role(Oid member, Oid role) { + Oid mdb_superuser_roleoid; + /* Fast path for simple case */ if (member == role) return true; @@ -5346,6 +5348,23 @@ has_privs_of_role(Oid member, Oid role) if (superuser_arg(member)) return true; + mdb_superuser_roleoid = get_role_oid("mdb_superuser", true /*if nodoby created mdb_superuser role in this database*/); + + if (is_member_of_role(member, mdb_superuser_roleoid)) { + /* if target role is superuser, disallow */ + if (!superuser_arg(role)) { + /* we want mdb_roles_admin to bypass + * has_priv_of_roles test + * if target role is neither superuser nor + * some dangerous system role + */ + if (!has_privs_of_unwanted_system_role(role)) { + return true; + } + } + } + + /* * Find all the roles that member has the privileges of, including * multi-level recursion, then see if target role is any one of them. @@ -5355,6 +5374,8 @@ has_privs_of_role(Oid member, Oid role) role); } +// -- mdb_superuser patch + // -- non-upstream patch begin /* * Is userId allowed to bypass ownership check @@ -5644,6 +5665,7 @@ select_best_grantor(Oid roleId, AclMode privileges, List *roles_list; int nrights; ListCell *l; + Oid mdb_superuser_roleoid; /* * The object owner is always treated as having all grant options, so if @@ -5658,6 +5680,16 @@ select_best_grantor(Oid roleId, AclMode privileges, return; } + mdb_superuser_roleoid = get_role_oid("mdb_superuser", true /*if nodoby created mdb_superuser role in this database*/); + + if (is_member_of_role(GetUserId(), mdb_superuser_roleoid) + && has_privs_of_role(GetUserId(), ownerId)) { + *grantorId = mdb_superuser_roleoid; + AclMode mdb_superuser_allowed_privs = needed_goptions; + *grantOptions = mdb_superuser_allowed_privs; + return; + } + /* * Otherwise we have to do a careful search to see if roleId has the * privileges of any suitable role. Note: we can hang onto the result of diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 14833363dd9..185f8bbea84 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -259,6 +259,9 @@ /* Define to 1 if you have the `numa' library (-lnuma). */ #undef HAVE_LIBNUMA +/* Define to 1 if you have the `mdblocales' library (-lmdblocales). */ +#undef HAVE_LIBMDBLOCALES + /* Define to 1 if you have the `pam' library (-lpam). */ #undef HAVE_LIBPAM @@ -723,6 +726,9 @@ /* Define to 1 to build with LZ4 support. (--with-lz4) */ #undef USE_LZ4 +/* Define to 1 to build with MDB locales. (--with-mdblocales) */ +#undef USE_MDBLOCALES + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/include/utils/acl.h b/src/include/utils/acl.h index 7cfd92da9dc..ae741a5964b 100644 --- a/src/include/utils/acl.h +++ b/src/include/utils/acl.h @@ -212,6 +212,7 @@ extern int aclmembers(const Acl *acl, Oid **roleids); extern bool has_privs_of_role(Oid member, Oid role); extern bool member_can_set_role(Oid member, Oid role); extern void check_can_set_role(Oid member, Oid role); +extern bool has_privs_of_role_strict(Oid member, Oid role); extern bool is_member_of_role(Oid member, Oid role); extern bool is_member_of_role_nosuper(Oid member, Oid role); extern bool is_admin_of_role(Oid member, Oid role); diff --git a/src/test/regress/expected/mdb_superuser.out b/src/test/regress/expected/mdb_superuser.out new file mode 100644 index 00000000000..e8dbe2fe1cc --- /dev/null +++ b/src/test/regress/expected/mdb_superuser.out @@ -0,0 +1,108 @@ +CREATE ROLE regress_mdb_superuser_user1; +CREATE ROLE regress_mdb_superuser_user2; +CREATE ROLE regress_mdb_superuser_user3; +GRANT mdb_admin TO mdb_superuser; +CREATE ROLE regress_superuser WITH SUPERUSER; +GRANT mdb_superuser TO regress_mdb_superuser_user1; +GRANT CREATE ON DATABASE regression TO regress_mdb_superuser_user2; +GRANT CREATE ON DATABASE regression TO regress_mdb_superuser_user3; +SET ROLE regress_mdb_superuser_user2; +CREATE FUNCTION regress_mdb_superuser_add(integer, integer) RETURNS integer + AS 'SELECT $1 + $2;' + LANGUAGE SQL + IMMUTABLE + RETURNS NULL ON NULL INPUT; +CREATE SCHEMA regress_mdb_superuser_schema; +CREATE TABLE regress_mdb_superuser_schema.regress_mdb_superuser_table(); +CREATE TABLE regress_mdb_superuser_table(); +CREATE VIEW regress_mdb_superuser_view as SELECT 1; +SET ROLE regress_mdb_superuser_user3; +INSERT INTO regress_mdb_superuser_table SELECT * FROM regress_mdb_superuser_table; +ERROR: permission denied for table regress_mdb_superuser_table +SET ROLE regress_mdb_superuser_user1; +-- mdb_superuser can grant to other role +GRANT USAGE, CREATE ON SCHEMA regress_mdb_superuser_schema TO regress_mdb_superuser_user3; +GRANT ALL PRIVILEGES ON TABLE regress_mdb_superuser_table TO regress_mdb_superuser_user3; +REVOKE ALL PRIVILEGES ON TABLE regress_mdb_superuser_table FROM regress_mdb_superuser_user3; +GRANT INSERT, SELECT ON TABLE regress_mdb_superuser_table TO regress_mdb_superuser_user3; +-- grant works +SET ROLE regress_mdb_superuser_user3; +INSERT INTO regress_mdb_superuser_table SELECT * FROM regress_mdb_superuser_table; +SET ROLE mdb_superuser; +-- mdb_superuser drop object of other role +DROP TABLE regress_mdb_superuser_table; +-- mdb admin fails to transfer ownership to superusers and system roles +RESET SESSION AUTHORIZATION; +CREATE TABLE regress_superuser_table(); +SET ROLE pg_read_server_files; +CREATE TABLE regress_pgrsf_table(); +SET ROLE pg_write_server_files; +CREATE TABLE regress_pgwsf_table(); +SET ROLE pg_execute_server_program; +CREATE TABLE regress_pgxsp_table(); +SET ROLE pg_read_all_data; +CREATE TABLE regress_pgrad_table(); +SET ROLE pg_write_all_data; +CREATE TABLE regress_pgrwd_table(); +SET ROLE mdb_superuser; +-- cannot read all data (fail) +SELECT * FROM pg_authid; +ERROR: permission denied for table pg_authid +-- can drop superuser objects, because has power of pg_database_owner +DROP TABLE regress_superuser_table; +DROP TABLE regress_pgrsf_table; +DROP TABLE regress_pgwsf_table; +DROP TABLE regress_pgxsp_table; +DROP TABLE regress_pgrad_table; +DROP TABLE regress_pgrwd_table; +-- does allowed to creare database, role or extension +-- or grant such priviledge +CREATE DATABASE regress_db_fail; +ERROR: permission denied to create database +CREATE ROLE regress_role_fail; +ERROR: permission denied to create role +DETAIL: Only roles with the CREATEROLE attribute may create roles. +ALTER ROLE mdb_superuser WITH CREATEROLE; +ERROR: permission denied to alter role +DETAIL: Only roles with the CREATEROLE attribute and the ADMIN option on role "mdb_superuser" may alter this role. +ALTER ROLE mdb_superuser WITH CREATEDB; +ERROR: permission denied to alter role +DETAIL: Only roles with the CREATEROLE attribute and the ADMIN option on role "mdb_superuser" may alter this role. +ALTER ROLE regress_mdb_superuser_user2 WITH CREATEROLE; +ERROR: permission denied to alter role +DETAIL: Only roles with the CREATEROLE attribute and the ADMIN option on role "regress_mdb_superuser_user2" may alter this role. +ALTER ROLE regress_mdb_superuser_user2 WITH CREATEDB; +ERROR: permission denied to alter role +DETAIL: Only roles with the CREATEROLE attribute and the ADMIN option on role "regress_mdb_superuser_user2" may alter this role. +-- mdb_superuser more powerfull than pg_database_owner +RESET SESSION AUTHORIZATION; +CREATE DATABASE regress_check_owner OWNER regress_mdb_superuser_user2; +\c regress_check_owner; +SET ROLE regress_mdb_superuser_user2; +CREATE SCHEMA regtest; +CREATE TABLE regtest.regtest(); +-- this should fail +SET ROLE regress_mdb_superuser_user3; +GRANT ALL ON TABLE regtest.regtest TO regress_mdb_superuser_user3; +ERROR: permission denied for schema regtest +ALTER TABLE regtest.regtest OWNER TO regress_mdb_superuser_user3; +ERROR: permission denied for schema regtest +SET ROLE regress_mdb_superuser_user1; +GRANT ALL ON TABLE regtest.regtest TO regress_mdb_superuser_user1; +ALTER TABLE regtest.regtest OWNER TO regress_mdb_superuser_user1; +\c regression +DROP DATABASE regress_check_owner; +-- end tests +RESET SESSION AUTHORIZATION; +-- +REVOKE CREATE ON DATABASE regression FROM regress_mdb_superuser_user2; +REVOKE CREATE ON DATABASE regression FROM regress_mdb_superuser_user3; +DROP VIEW regress_mdb_superuser_view; +DROP FUNCTION regress_mdb_superuser_add; +DROP TABLE regress_mdb_superuser_schema.regress_mdb_superuser_table; +DROP TABLE regress_mdb_superuser_table; +ERROR: table "regress_mdb_superuser_table" does not exist +DROP SCHEMA regress_mdb_superuser_schema; +DROP ROLE regress_mdb_superuser_user1; +DROP ROLE regress_mdb_superuser_user2; +DROP ROLE regress_mdb_superuser_user3; diff --git a/src/test/regress/sql/mdb_superuser.sql b/src/test/regress/sql/mdb_superuser.sql new file mode 100644 index 00000000000..77dc489f715 --- /dev/null +++ b/src/test/regress/sql/mdb_superuser.sql @@ -0,0 +1,138 @@ +CREATE ROLE regress_mdb_superuser_user1; +CREATE ROLE regress_mdb_superuser_user2; +CREATE ROLE regress_mdb_superuser_user3; + +GRANT mdb_admin TO mdb_superuser; + +CREATE ROLE regress_superuser WITH SUPERUSER; + +GRANT mdb_superuser TO regress_mdb_superuser_user1; + +GRANT CREATE ON DATABASE regression TO regress_mdb_superuser_user2; +GRANT CREATE ON DATABASE regression TO regress_mdb_superuser_user3; + + +SET ROLE regress_mdb_superuser_user2; + +CREATE FUNCTION regress_mdb_superuser_add(integer, integer) RETURNS integer + AS 'SELECT $1 + $2;' + LANGUAGE SQL + IMMUTABLE + RETURNS NULL ON NULL INPUT; + +CREATE SCHEMA regress_mdb_superuser_schema; +CREATE TABLE regress_mdb_superuser_schema.regress_mdb_superuser_table(); +CREATE TABLE regress_mdb_superuser_table(); +CREATE VIEW regress_mdb_superuser_view as SELECT 1; + +SET ROLE regress_mdb_superuser_user3; +INSERT INTO regress_mdb_superuser_table SELECT * FROM regress_mdb_superuser_table; + +SET ROLE regress_mdb_superuser_user1; + +-- mdb_superuser can grant to other role +GRANT USAGE, CREATE ON SCHEMA regress_mdb_superuser_schema TO regress_mdb_superuser_user3; +GRANT ALL PRIVILEGES ON TABLE regress_mdb_superuser_table TO regress_mdb_superuser_user3; +REVOKE ALL PRIVILEGES ON TABLE regress_mdb_superuser_table FROM regress_mdb_superuser_user3; + +GRANT INSERT, SELECT ON TABLE regress_mdb_superuser_table TO regress_mdb_superuser_user3; + +-- grant works +SET ROLE regress_mdb_superuser_user3; +INSERT INTO regress_mdb_superuser_table SELECT * FROM regress_mdb_superuser_table; + +SET ROLE mdb_superuser; + +-- mdb_superuser drop object of other role +DROP TABLE regress_mdb_superuser_table; +-- mdb admin fails to transfer ownership to superusers and system roles + +RESET SESSION AUTHORIZATION; + +CREATE TABLE regress_superuser_table(); + +SET ROLE pg_read_server_files; + +CREATE TABLE regress_pgrsf_table(); + +SET ROLE pg_write_server_files; + +CREATE TABLE regress_pgwsf_table(); + +SET ROLE pg_execute_server_program; + +CREATE TABLE regress_pgxsp_table(); + +SET ROLE pg_read_all_data; + +CREATE TABLE regress_pgrad_table(); + +SET ROLE pg_write_all_data; + +CREATE TABLE regress_pgrwd_table(); + +SET ROLE mdb_superuser; + +-- cannot read all data (fail) +SELECT * FROM pg_authid; + +-- can drop superuser objects, because has power of pg_database_owner +DROP TABLE regress_superuser_table; +DROP TABLE regress_pgrsf_table; +DROP TABLE regress_pgwsf_table; +DROP TABLE regress_pgxsp_table; +DROP TABLE regress_pgrad_table; +DROP TABLE regress_pgrwd_table; + + +-- does allowed to creare database, role or extension +-- or grant such priviledge + +CREATE DATABASE regress_db_fail; +CREATE ROLE regress_role_fail; + +ALTER ROLE mdb_superuser WITH CREATEROLE; +ALTER ROLE mdb_superuser WITH CREATEDB; + +ALTER ROLE regress_mdb_superuser_user2 WITH CREATEROLE; +ALTER ROLE regress_mdb_superuser_user2 WITH CREATEDB; + +-- mdb_superuser more powerfull than pg_database_owner + +RESET SESSION AUTHORIZATION; +CREATE DATABASE regress_check_owner OWNER regress_mdb_superuser_user2; + +\c regress_check_owner; + +SET ROLE regress_mdb_superuser_user2; +CREATE SCHEMA regtest; +CREATE TABLE regtest.regtest(); + +-- this should fail + +SET ROLE regress_mdb_superuser_user3; +GRANT ALL ON TABLE regtest.regtest TO regress_mdb_superuser_user3; +ALTER TABLE regtest.regtest OWNER TO regress_mdb_superuser_user3; + +SET ROLE regress_mdb_superuser_user1; +GRANT ALL ON TABLE regtest.regtest TO regress_mdb_superuser_user1; +ALTER TABLE regtest.regtest OWNER TO regress_mdb_superuser_user1; + +\c regression +DROP DATABASE regress_check_owner; + +-- end tests + +RESET SESSION AUTHORIZATION; +-- +REVOKE CREATE ON DATABASE regression FROM regress_mdb_superuser_user2; +REVOKE CREATE ON DATABASE regression FROM regress_mdb_superuser_user3; + +DROP VIEW regress_mdb_superuser_view; +DROP FUNCTION regress_mdb_superuser_add; +DROP TABLE regress_mdb_superuser_schema.regress_mdb_superuser_table; +DROP TABLE regress_mdb_superuser_table; +DROP SCHEMA regress_mdb_superuser_schema; +DROP ROLE regress_mdb_superuser_user1; +DROP ROLE regress_mdb_superuser_user2; +DROP ROLE regress_mdb_superuser_user3; From 02b91a60c5fec0a4ca1a19274e66114be63550b1 Mon Sep 17 00:00:00 2001 From: reshke Date: Tue, 22 Feb 2022 23:26:51 +0300 Subject: [PATCH 14/45] provide [mdb -postgresql] restict grant roles in YC[MDB-16990] --- src/backend/access/common/Makefile | 3 +- src/backend/access/common/yc_checker.c | 21 +++++++++++++ src/backend/commands/user.c | 20 ++++++++++++ src/backend/utils/misc/guc.c | 1 + src/backend/utils/misc/guc_tables.c | 21 +++++++++++++ src/backend/utils/misc/postgresql.conf.sample | 2 ++ src/include/access/yc_checker.h | 31 +++++++++++++++++++ src/include/utils/acl.h | 1 - src/test/modules/test_misc/t/003_check_guc.pl | 2 ++ 9 files changed, 100 insertions(+), 2 deletions(-) create mode 100644 src/backend/access/common/yc_checker.c create mode 100644 src/include/access/yc_checker.h diff --git a/src/backend/access/common/Makefile b/src/backend/access/common/Makefile index e78de312659..399e7e602ce 100644 --- a/src/backend/access/common/Makefile +++ b/src/backend/access/common/Makefile @@ -29,6 +29,7 @@ OBJS = \ toast_compression.o \ toast_internals.o \ tupconvert.o \ - tupdesc.o + tupdesc.o \ + yc_checker.o include $(top_srcdir)/src/backend/common.mk diff --git a/src/backend/access/common/yc_checker.c b/src/backend/access/common/yc_checker.c new file mode 100644 index 00000000000..a7e7822cc2d --- /dev/null +++ b/src/backend/access/common/yc_checker.c @@ -0,0 +1,21 @@ +/*------------------------------------------------------------------------- + * + * yc_checker.c + * yc routines + * + * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * + * IDENTIFICATION + * src/backend/access/common/yc_checker.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" +#include "access/yc_checker.h" + +/* GUC variables */ + +YCGrantCheckerType yc_grant_checker_type = YC_GRANT_CHECKER_OFF; diff --git a/src/backend/commands/user.c b/src/backend/commands/user.c index 5ce8ffa863a..2f070d1abcc 100644 --- a/src/backend/commands/user.c +++ b/src/backend/commands/user.c @@ -16,6 +16,7 @@ #include "access/htup_details.h" #include "access/table.h" #include "access/xact.h" +#include "access/yc_checker.h" #include "catalog/binary_upgrade.h" #include "catalog/catalog.h" #include "catalog/dependency.h" @@ -1689,6 +1690,25 @@ AddRoleMems(Oid currentUserId, const char *rolename, Oid roleid, Assert(list_length(memberSpecs) == list_length(memberIds)); + if (!superuser()) { + switch (yc_grant_checker_type) { + case YC_GRANT_CHECKER_WARN: + ereport(WARNING, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("Roles granted using SQL might be eventually revoked. Please, use Yandex Cloud console, cli or terraform to grant role in postgresql cluster."))); + break; + case YC_GRANT_CHECKER_CRIT: + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("Roles granted using SQL might be eventually revoked. Please, use Yandex Cloud console, cli or terraform to grant role in postgresql cluster."))); + break; + case YC_GRANT_CHECKER_OFF: + default: + break; + + } + } + /* Validate grantor (and resolve implicit grantor if not specified). */ grantorId = check_role_grantor(currentUserId, roleid, grantorId, true); diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index ce5449f2878..7f982236d2e 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -3413,6 +3413,7 @@ set_config_with_handle(const char *name, config_handle *handle, void *newextra = NULL; bool prohibitValueChange = false; bool makeDefault; + Oid role; if (elevel == 0) { diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index 5b0cdc70942..2155221f7a0 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -106,6 +106,9 @@ #ifdef TRACE_SYNCSCAN #include "access/syncscan.h" #endif +/* MDB patch */ +#include "access/yc_checker.h" +/**/ /* This value is normally passed in from the Makefile */ #ifndef PG_KRB_SRVTAB @@ -500,6 +503,14 @@ static const struct config_enum_entry file_extend_method_options[] = { {NULL, 0, false} }; +static const struct config_enum_entry yc_grant_checker_options[] = { + {"off", YC_GRANT_CHECKER_OFF, false}, + {"warn", YC_GRANT_CHECKER_WARN, false}, + {"crit", YC_GRANT_CHECKER_CRIT, false}, + {NULL, 0, false} +}; + + /* * Options for enum values stored in other modules */ @@ -5076,6 +5087,16 @@ struct config_enum ConfigureNamesEnum[] = NULL, NULL, NULL }, + { + {"ycmdb.yc_grant_checker", PGC_SUSET, CLIENT_CONN_STATEMENT, + gettext_noop("Enables YC MDB runtime checker, which check if user is ok to grant roles to other users."), + NULL + }, + ((int *) &yc_grant_checker_type), + YC_GRANT_CHECKER_OFF, + yc_grant_checker_options, + NULL, NULL, NULL + }, { {"default_transaction_isolation", PGC_USERSET, CLIENT_CONN_STATEMENT, gettext_noop("Sets the transaction isolation level of each new transaction."), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index f1a862b9b8c..5e75c106405 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -886,6 +886,8 @@ #synchronous_commit_cancelation = off +#ycmdb.yc_grant_checker = off + #------------------------------------------------------------------------------ # CUSTOMIZED OPTIONS #------------------------------------------------------------------------------ diff --git a/src/include/access/yc_checker.h b/src/include/access/yc_checker.h new file mode 100644 index 00000000000..9c9a6b6ad4c --- /dev/null +++ b/src/include/access/yc_checker.h @@ -0,0 +1,31 @@ +# +/*------------------------------------------------------------------------- + * + * yc_checker.h + * + * Header file for YC MDB specific only GUC variables, + * + * Portions Copyright (c) 1996-2021, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/storage/yc_checker.h + * + *------------------------------------------------------------------------- + */ +#ifndef PG_YC_CHECKER_H +#define PG_YC_CHECKER_H + +/* Possible values for yc_grant_checker_type */ +typedef enum +{ + YC_GRANT_CHECKER_OFF, + YC_GRANT_CHECKER_WARN, + YC_GRANT_CHECKER_CRIT, +} YCGrantCheckerType; + +/* GUC variables */ + +extern PGDLLIMPORT YCGrantCheckerType yc_grant_checker_type; + + +#endif /* PG_YC_CHECKER_H */ diff --git a/src/include/utils/acl.h b/src/include/utils/acl.h index ae741a5964b..7cfd92da9dc 100644 --- a/src/include/utils/acl.h +++ b/src/include/utils/acl.h @@ -212,7 +212,6 @@ extern int aclmembers(const Acl *acl, Oid **roleids); extern bool has_privs_of_role(Oid member, Oid role); extern bool member_can_set_role(Oid member, Oid role); extern void check_can_set_role(Oid member, Oid role); -extern bool has_privs_of_role_strict(Oid member, Oid role); extern bool is_member_of_role(Oid member, Oid role); extern bool is_member_of_role_nosuper(Oid member, Oid role); extern bool is_admin_of_role(Oid member, Oid role); diff --git a/src/test/modules/test_misc/t/003_check_guc.pl b/src/test/modules/test_misc/t/003_check_guc.pl index 5ae23192a47..1df531b03ef 100644 --- a/src/test/modules/test_misc/t/003_check_guc.pl +++ b/src/test/modules/test_misc/t/003_check_guc.pl @@ -72,6 +72,8 @@ } } +push @gucs_in_file, "ycmdb.yc_grant_checker"; + close $contents; # Cross-check that all the GUCs found in the sample file match the ones From 98b85cfc63efcc3def502c3e0da2b0c694367280 Mon Sep 17 00:00:00 2001 From: reshke Date: Wed, 11 May 2022 23:11:42 +0300 Subject: [PATCH 15/45] MDB-16955 : disallow to kill repl mon in cloud --- src/backend/postmaster/bgworker.c | 28 +++++++++++++++++++++++++++ src/backend/storage/ipc/signalfuncs.c | 11 +++++++++++ src/include/postmaster/bgworker.h | 1 + 3 files changed, 40 insertions(+) diff --git a/src/backend/postmaster/bgworker.c b/src/backend/postmaster/bgworker.c index 1ad65c237c3..c602fd89560 100644 --- a/src/backend/postmaster/bgworker.c +++ b/src/backend/postmaster/bgworker.c @@ -1396,3 +1396,31 @@ GetBackgroundWorkerTypeByPid(pid_t pid) return result; } + + +bool +GetBackgroundWorkerFindByPidCmp(pid_t pid, const char *target) +{ + int slotno; + bool result = false; + + LWLockAcquire(BackgroundWorkerLock, LW_SHARED); + + for (slotno = 0; slotno < BackgroundWorkerData->total_slots; slotno++) + { + BackgroundWorkerSlot *slot = &BackgroundWorkerData->slot[slotno]; + + if (slot->pid > 0 && slot->pid == pid) + { + if (slot->worker.bgw_type) { + result = strcmp(target, slot->worker.bgw_type) == 0; + } else { + result = false; + } + break; + } + } + + LWLockRelease(BackgroundWorkerLock); + return result; +} diff --git a/src/backend/storage/ipc/signalfuncs.c b/src/backend/storage/ipc/signalfuncs.c index 7401d30d091..3c74ba9b3cb 100644 --- a/src/backend/storage/ipc/signalfuncs.c +++ b/src/backend/storage/ipc/signalfuncs.c @@ -106,6 +106,17 @@ pg_signal_backend(int pid, int sig) !has_privs_of_role(GetUserId(), ROLE_PG_SIGNAL_BACKEND)) return SIGNAL_BACKEND_NOPERMISSION; + if (!superuser_arg(GetUserId()) && local_beentry != NULL) { + PgBackendStatus *beentry = &local_beentry->backendStatus; + // MDB-16955 : disallow to kill repl mon in cloud + if (beentry != NULL && beentry->st_backendType == B_BG_WORKER) + { + if (GetBackgroundWorkerFindByPidCmp(beentry->st_procpid, "repl_mon")) { + return SIGNAL_BACKEND_NOPERMISSION; + } + } + } + /* * Can the process we just validated above end, followed by the pid being * recycled for a new process, before reaching here? Then we'd be trying diff --git a/src/include/postmaster/bgworker.h b/src/include/postmaster/bgworker.h index 058667a47a0..619c2e51dad 100644 --- a/src/include/postmaster/bgworker.h +++ b/src/include/postmaster/bgworker.h @@ -125,6 +125,7 @@ extern BgwHandleStatus WaitForBackgroundWorkerStartup(BackgroundWorkerHandle *ha extern BgwHandleStatus WaitForBackgroundWorkerShutdown(BackgroundWorkerHandle *); extern const char *GetBackgroundWorkerTypeByPid(pid_t pid); +extern bool GetBackgroundWorkerFindByPidCmp(pid_t pid, const char *target); /* Terminate a bgworker */ extern void TerminateBackgroundWorker(BackgroundWorkerHandle *handle); From 946a8f9276b303a75b81d55aa0d278dcbd66ab4a Mon Sep 17 00:00:00 2001 From: "Andrey M. Borodin" Date: Mon, 12 Sep 2022 11:47:30 +0500 Subject: [PATCH 16/45] Demonstrate and fix lock of all SQL queries by pg_stat_statements --- .../pg_stat_statements/pg_stat_statements.c | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/contrib/pg_stat_statements/pg_stat_statements.c b/contrib/pg_stat_statements/pg_stat_statements.c index 1c07acf231e..77450947bee 100644 --- a/contrib/pg_stat_statements/pg_stat_statements.c +++ b/contrib/pg_stat_statements/pg_stat_statements.c @@ -1325,8 +1325,12 @@ pgss_store(const char *query, int64 queryId, key.queryid = queryId; key.toplevel = (nesting_level == 0); - /* Lookup the hash table entry with shared lock. */ - LWLockAcquire(pgss->lock, LW_SHARED); + /* + * Lookup the hash table entry with shared lock. + * If exclusive lock is taken - just give up. + */ + if (!LWLockConditionalAcquire(pgss->lock, LW_SHARED)) + return; entry = (pgssEntry *) hash_search(pgss_hash, &key, HASH_FIND, NULL); @@ -1351,7 +1355,13 @@ pgss_store(const char *query, int64 queryId, norm_query = generate_normalized_query(jstate, query, query_location, &query_len); - LWLockAcquire(pgss->lock, LW_SHARED); + /* exclusive lock may be taken while we were doing this */ + /* XXX: Andrey: I'm not sure we should drop here shared lock at all */ + if (!LWLockConditionalAcquire(pgss->lock, LW_SHARED)) + { + pfree(norm_query); + return; + } } /* Append new query text to file with only shared lock held */ @@ -1367,7 +1377,9 @@ pgss_store(const char *query, int64 queryId, /* Need exclusive lock to make a new hashtable entry - promote */ LWLockRelease(pgss->lock); - LWLockAcquire(pgss->lock, LW_EXCLUSIVE); + /* This renders impossible to enter another concurrent query */ + if (!LWLockConditionalAcquire(pgss->lock, LW_EXCLUSIVE)) + return; /* * A garbage collection may have occurred while we weren't holding the From 8fc0b560aa3bdbca1f672db6368a2cef44ae9696 Mon Sep 17 00:00:00 2001 From: reshke kirill Date: Wed, 18 Jan 2023 18:27:40 +0000 Subject: [PATCH 17/45] MDB-21297: forbit usage of COPY TO PROGRAMM and COPY FROM PROGRAMM to non-superuser It is well known that some of PostgreSQL-related security issues (CVE) was related to COPY FROM/TO PROGRAM exploits for priviledge escalation or other unwanted behaviour or consequences. Thus, proper usage of this feature needed. For now, simply forbit this. Add mdb copy test --- src/backend/commands/copy.c | 29 ++++++++++++------------ src/test/regress/expected/mdb_copy.out | 29 ++++++++++++++++++++++++ src/test/regress/parallel_schedule | 2 ++ src/test/regress/sql/mdb_copy.sql | 31 ++++++++++++++++++++++++++ 4 files changed, 77 insertions(+), 14 deletions(-) create mode 100644 src/test/regress/expected/mdb_copy.out create mode 100644 src/test/regress/sql/mdb_copy.sql diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c index 13ca317c991..975930f4a03 100644 --- a/src/backend/commands/copy.c +++ b/src/backend/commands/copy.c @@ -78,37 +78,38 @@ DoCopy(ParseState *pstate, const CopyStmt *stmt, { if (stmt->is_program) { - if (!has_privs_of_role(GetUserId(), ROLE_PG_EXECUTE_SERVER_PROGRAM)) - ereport(ERROR, - (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), - errmsg("permission denied to COPY to or from an external program"), - errdetail("Only roles with privileges of the \"%s\" role may COPY to or from an external program.", - "pg_execute_server_program"), - errhint("Anyone can COPY to stdout or from stdin. " - "psql's \\copy command also works for anyone."))); + // -- non-upstream patch begin + /* + * MDB-21297: forbit usage of COPY TO PROGRAM and COPY FROM PROGRAM to non-su + */ + + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("forbidden to COPY to or from an external program or file in Yandex Cloud"), + errhint("Anyone can COPY to stdout or from stdin. " + "psql's \\copy command also works for anyone."))); + + // --- non-upstream patch end } else { if (is_from && !has_privs_of_role(GetUserId(), ROLE_PG_READ_SERVER_FILES)) ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), - errmsg("permission denied to COPY from a file"), - errdetail("Only roles with privileges of the \"%s\" role may COPY from a file.", - "pg_read_server_files"), + errmsg("must be superuser or have privileges of the pg_read_server_files role to COPY from a file"), errhint("Anyone can COPY to stdout or from stdin. " "psql's \\copy command also works for anyone."))); if (!is_from && !has_privs_of_role(GetUserId(), ROLE_PG_WRITE_SERVER_FILES)) ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), - errmsg("permission denied to COPY to a file"), - errdetail("Only roles with privileges of the \"%s\" role may COPY to a file.", - "pg_write_server_files"), + errmsg("must be superuser or have privileges of the pg_write_server_files role to COPY to a file"), errhint("Anyone can COPY to stdout or from stdin. " "psql's \\copy command also works for anyone."))); } } + if (stmt->relation) { LOCKMODE lockmode = is_from ? RowExclusiveLock : AccessShareLock; diff --git a/src/test/regress/expected/mdb_copy.out b/src/test/regress/expected/mdb_copy.out new file mode 100644 index 00000000000..df6d70d1e7d --- /dev/null +++ b/src/test/regress/expected/mdb_copy.out @@ -0,0 +1,29 @@ +CREATE ROLE regress_mdb_copy_r1 LOGIN NOSUPERUSER; +CREATE ROLE regress_mdb_copy_r1_mdb_adm LOGIN NOSUPERUSER; +GRANT mdb_admin TO regress_mdb_copy_r1_mdb_adm; +CREATE ROLE regress_mdb_copy_r1_su LOGIN SUPERUSER; +-- should fail +SET ROLE regress_mdb_copy_r1; +CREATE TABLE tt(i int); +COPY tt FROM PROGRAM '/bin/bash'; +ERROR: forbidden to COPY to or from an external program or file in Yandex Cloud +HINT: Anyone can COPY to stdout or from stdin. psql's \copy command also works for anyone. +DROP TABLE tt; +-- should fail +SET ROLE regress_mdb_copy_r1_mdb_adm; +CREATE TABLE tt(i int); +COPY tt FROM PROGRAM '/bin/bash'; +ERROR: forbidden to COPY to or from an external program or file in Yandex Cloud +HINT: Anyone can COPY to stdout or from stdin. psql's \copy command also works for anyone. +DROP TABLE tt; +-- fail, no one can do it +SET ROLE regress_mdb_copy_r1_su; +CREATE TABLE tt(i int); +COPY tt FROM PROGRAM '/bin/bash'; +ERROR: forbidden to COPY to or from an external program or file in Yandex Cloud +HINT: Anyone can COPY to stdout or from stdin. psql's \copy command also works for anyone. +DROP TABLE tt; +RESET SESSION AUTHORIZATION; +DROP ROLE regress_mdb_copy_r1; +DROP ROLE regress_mdb_copy_r1_mdb_adm; +DROP ROLE regress_mdb_copy_r1_su; diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule index 59abe787bde..f0de2bca64f 100644 --- a/src/test/regress/parallel_schedule +++ b/src/test/regress/parallel_schedule @@ -19,6 +19,8 @@ test: mdb_superuser test: mdb_replication +test: mdb_copy + # ---------- # The first group of parallel tests # ---------- diff --git a/src/test/regress/sql/mdb_copy.sql b/src/test/regress/sql/mdb_copy.sql new file mode 100644 index 00000000000..c734b2814c5 --- /dev/null +++ b/src/test/regress/sql/mdb_copy.sql @@ -0,0 +1,31 @@ +CREATE ROLE regress_mdb_copy_r1 LOGIN NOSUPERUSER; +CREATE ROLE regress_mdb_copy_r1_mdb_adm LOGIN NOSUPERUSER; +GRANT mdb_admin TO regress_mdb_copy_r1_mdb_adm; +CREATE ROLE regress_mdb_copy_r1_su LOGIN SUPERUSER; + +-- should fail +SET ROLE regress_mdb_copy_r1; + +CREATE TABLE tt(i int); +COPY tt FROM PROGRAM '/bin/bash'; +DROP TABLE tt; + +-- should fail +SET ROLE regress_mdb_copy_r1_mdb_adm; + +CREATE TABLE tt(i int); +COPY tt FROM PROGRAM '/bin/bash'; +DROP TABLE tt; + +-- fail, no one can do it +SET ROLE regress_mdb_copy_r1_su; + +CREATE TABLE tt(i int); +COPY tt FROM PROGRAM '/bin/bash'; +DROP TABLE tt; + +RESET SESSION AUTHORIZATION; + +DROP ROLE regress_mdb_copy_r1; +DROP ROLE regress_mdb_copy_r1_mdb_adm; +DROP ROLE regress_mdb_copy_r1_su; From 7722d81a7abd1e1df5dbd7d676cbad13e92f08dc Mon Sep 17 00:00:00 2001 From: usernamedt Date: Mon, 13 Feb 2023 15:00:31 +0800 Subject: [PATCH 18/45] Implement mdb-locales patch Do not use mdb_locales and mdb_newlocale when configured without them. Added ifdef codepath build without mdb-locales feature Add mdb locales patch, restore COPY from/to files, enable regress. Squashed commit of the following: commit 9f8ea4a5f42e0fd6077061bafbb428e88499896f Author: reshke kirill Date: Wed Feb 22 09:27:53 2023 +0000 Add mdb locales patch, restore COPY from/to files, enable regress. This commit does several things. * Enables back COPY from/to FILE functionality, becuase it is used by pg_regress * Enables pg_regress tests in deb build itself. * Add mdb locales function and checks that package build with mdb-locales support Add configure ac target to define USE_MDBLOCALES properly Refactor optional setlocale, fix minor issues Add missing SETLOCALE and NEWLOCALE calls. They seem to be missing after 37851a8b83d3 and 4f5cef2607c1 Co-authored-by: reshke reshke@double.cloud --- configure | 127 +++++++++++++++++++++-- configure.ac | 17 +++ src/backend/utils/adt/Makefile | 3 +- src/backend/utils/adt/mdb.c | 36 +++++++ src/backend/utils/adt/pg_locale.c | 13 +-- src/backend/utils/adt/pg_locale_libc.c | 10 +- src/backend/utils/mb/mbutils.c | 3 +- src/bin/initdb/initdb.c | 11 +- src/bin/pg_upgrade/check.c | 2 + src/common/exec.c | 4 +- src/include/catalog/pg_proc.dat | 7 +- src/include/common/mdb_locale.h | 24 +++++ src/include/pg_config.h.in | 3 + src/interfaces/ecpg/ecpglib/connect.c | 3 +- src/interfaces/ecpg/ecpglib/descriptor.c | 8 +- src/interfaces/ecpg/ecpglib/execute.c | 7 +- src/interfaces/libpq/Makefile | 4 +- src/pl/plperl/plperl.c | 19 ++-- src/port/chklocale.c | 6 +- src/port/pg_localeconv_r.c | 10 +- src/port/win32setlocale.c | 2 +- src/test/locale/test-ctype.c | 4 +- src/test/regress/expected/misc.out | 7 ++ src/test/regress/sql/misc.sql | 5 + 24 files changed, 282 insertions(+), 53 deletions(-) create mode 100644 src/backend/utils/adt/mdb.c create mode 100644 src/include/common/mdb_locale.h diff --git a/configure b/configure index 2f1377b7ee0..da9a74d77e8 100755 --- a/configure +++ b/configure @@ -685,6 +685,7 @@ BISON MKDIR_P LN_S TAR +USE_MDBLOCALES install_bin INSTALL_DATA INSTALL_SCRIPT @@ -815,6 +816,7 @@ infodir docdir oldincludedir includedir +runstatedir localstatedir sharedstatedir sysconfdir @@ -885,6 +887,8 @@ with_system_tzdata with_zlib with_lz4 with_zstd +with_gnu_ld +with_mdblocales with_ssl with_openssl enable_largefile @@ -966,6 +970,7 @@ datadir='${datarootdir}' sysconfdir='${prefix}/etc' sharedstatedir='${prefix}/com' localstatedir='${prefix}/var' +runstatedir='${localstatedir}/run' includedir='${prefix}/include' oldincludedir='/usr/include' docdir='${datarootdir}/doc/${PACKAGE_TARNAME}' @@ -1218,6 +1223,15 @@ do | -silent | --silent | --silen | --sile | --sil) silent=yes ;; + -runstatedir | --runstatedir | --runstatedi | --runstated \ + | --runstate | --runstat | --runsta | --runst | --runs \ + | --run | --ru | --r) + ac_prev=runstatedir ;; + -runstatedir=* | --runstatedir=* | --runstatedi=* | --runstated=* \ + | --runstate=* | --runstat=* | --runsta=* | --runst=* | --runs=* \ + | --run=* | --ru=* | --r=*) + runstatedir=$ac_optarg ;; + -sbindir | --sbindir | --sbindi | --sbind | --sbin | --sbi | --sb) ac_prev=sbindir ;; -sbindir=* | --sbindir=* | --sbindi=* | --sbind=* | --sbin=* \ @@ -1355,7 +1369,7 @@ fi for ac_var in exec_prefix prefix bindir sbindir libexecdir datarootdir \ datadir sysconfdir sharedstatedir localstatedir includedir \ oldincludedir docdir infodir htmldir dvidir pdfdir psdir \ - libdir localedir mandir + libdir localedir mandir runstatedir do eval ac_val=\$$ac_var # Remove trailing slashes. @@ -1508,6 +1522,7 @@ Fine tuning of the installation directories: --sysconfdir=DIR read-only single-machine data [PREFIX/etc] --sharedstatedir=DIR modifiable architecture-independent data [PREFIX/com] --localstatedir=DIR modifiable single-machine data [PREFIX/var] + --runstatedir=DIR modifiable per-process data [LOCALSTATEDIR/run] --libdir=DIR object code libraries [EPREFIX/lib] --includedir=DIR C header files [PREFIX/include] --oldincludedir=DIR C header files for non-gcc [/usr/include] @@ -1605,6 +1620,8 @@ Optional Packages: --without-zlib do not use Zlib --with-lz4 build with LZ4 support --with-zstd build with ZSTD support + --with-gnu-ld assume the C compiler uses GNU ld [default=no] + --without-mdblocales build without MDB locales --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -2828,7 +2845,6 @@ ac_compiler_gnu=$ac_cv_c_compiler_gnu - ac_aux_dir= for ac_dir in config "$srcdir"/config; do if test -f "$ac_dir/install-sh"; then @@ -2859,6 +2875,7 @@ ac_configure="$SHELL $ac_aux_dir/configure" # Please don't use this var. +ac_configure_args=$(echo "$ac_configure_args" | sed -e "s/ -f\(debug\|file\)-prefix-map=[^' ]*//g") cat >>confdefs.h <<_ACEOF #define CONFIGURE_ARGS "$ac_configure_args" @@ -10236,6 +10253,40 @@ case $INSTALL in esac +# +# MDB locales +# + + + + +# Check whether --with-mdblocales was given. +if test "${with_mdblocales+set}" = set; then : + withval=$with_mdblocales; + case $withval in + yes) + +$as_echo "#define USE_MDBLOCALES 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-mdblocales option" "$LINENO" 5 + ;; + esac + +else + with_mdblocales=yes + +$as_echo "#define USE_MDBLOCALES 1" >>confdefs.h + +fi + + + + if test -z "$TAR"; then for ac_prog in tar do @@ -12715,8 +12766,57 @@ fi fi -if test "$with_libcurl" = yes ; then +if test "$with_mdblocales" = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for mdb_setlocale in -lmdblocales" >&5 +$as_echo_n "checking for mdb_setlocale in -lmdblocales... " >&6; } +if ${ac_cv_lib_mdblocales_mdb_setlocale+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_check_lib_save_LIBS=$LIBS +LIBS="-lmdblocales $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char mdb_setlocale (); +int +main () +{ +return mdb_setlocale (); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + ac_cv_lib_mdblocales_mdb_setlocale=yes +else + ac_cv_lib_mdblocales_mdb_setlocale=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_mdblocales_mdb_setlocale" >&5 +$as_echo "$ac_cv_lib_mdblocales_mdb_setlocale" >&6; } +if test "x$ac_cv_lib_mdblocales_mdb_setlocale" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LIBMDBLOCALES 1 +_ACEOF + + LIBS="-lmdblocales $LIBS" + +else + as_fn_error $? "mdblocales library not found" "$LINENO" 5 +fi + +fi + +if test "$with_libcurl" = yes ; then # libcurl compiler/linker flags are kept separate from the global flags, so # they have to be added back temporarily for the following tests. pgac_save_CPPFLAGS=$CPPFLAGS @@ -14125,6 +14225,17 @@ else fi +fi + +if test "$with_mdblocales" = yes; then + ac_fn_c_check_header_mongrel "$LINENO" "mdblocales.h" "ac_cv_header_mdblocales_h" "$ac_includes_default" +if test "x$ac_cv_header_mdblocales_h" = xyes; then : + +else + as_fn_error $? "mdblocales header not found." "$LINENO" 5 +fi + + fi if test "$with_gssapi" = yes ; then @@ -15461,7 +15572,7 @@ else We can't simply define LARGE_OFF_T to be 9223372036854775807, since some C++ compilers masquerading as C compilers incorrectly reject 9223372036854775807. */ -#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62)) +#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31)) int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721 && LARGE_OFF_T % 2147483647 == 1) ? 1 : -1]; @@ -15507,7 +15618,7 @@ else We can't simply define LARGE_OFF_T to be 9223372036854775807, since some C++ compilers masquerading as C compilers incorrectly reject 9223372036854775807. */ -#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62)) +#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31)) int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721 && LARGE_OFF_T % 2147483647 == 1) ? 1 : -1]; @@ -15531,7 +15642,7 @@ rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext We can't simply define LARGE_OFF_T to be 9223372036854775807, since some C++ compilers masquerading as C compilers incorrectly reject 9223372036854775807. */ -#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62)) +#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31)) int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721 && LARGE_OFF_T % 2147483647 == 1) ? 1 : -1]; @@ -15576,7 +15687,7 @@ else We can't simply define LARGE_OFF_T to be 9223372036854775807, since some C++ compilers masquerading as C compilers incorrectly reject 9223372036854775807. */ -#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62)) +#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31)) int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721 && LARGE_OFF_T % 2147483647 == 1) ? 1 : -1]; @@ -15600,7 +15711,7 @@ rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext We can't simply define LARGE_OFF_T to be 9223372036854775807, since some C++ compilers masquerading as C compilers incorrectly reject 9223372036854775807. */ -#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62)) +#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31)) int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721 && LARGE_OFF_T % 2147483647 == 1) ? 1 : -1]; diff --git a/configure.ac b/configure.ac index 6266944a32e..6555179690b 100644 --- a/configure.ac +++ b/configure.ac @@ -1209,6 +1209,14 @@ case $INSTALL in esac AC_SUBST(install_bin) +# +# MDB locales +# + +PGAC_ARG_BOOL(with, mdblocales, yes, [build without MDB locales], + [AC_DEFINE([USE_MDBLOCALES], 1, [Define to 1 to build with MDB locales. (--with-mdblocales)])]) +AC_SUBST(USE_MDBLOCALES) + PGAC_PATH_PROGS(TAR, tar) AC_PROG_LN_S AC_PROG_MKDIR_P @@ -1363,6 +1371,11 @@ if test "$with_libcurl" = yes ; then PGAC_CHECK_LIBCURL fi +if test "$with_mdblocales" = yes; then + AC_CHECK_LIB(mdblocales, mdb_setlocale, [], + [AC_MSG_ERROR([mdblocales library not found])]) +fi + if test "$with_gssapi" = yes ; then if test "$PORTNAME" != "win32"; then AC_SEARCH_LIBS(gss_store_cred_into, [gssapi_krb5 gss 'gssapi -lkrb5 -lcrypto'], [], @@ -1574,6 +1587,10 @@ if test "$with_zstd" = yes; then AC_CHECK_HEADER(zstd.h, [], [AC_MSG_ERROR([zstd.h header file is required for ZSTD])]) fi +if test "$with_mdblocales" = yes; then + AC_CHECK_HEADER(mdblocales.h, [], [AC_MSG_ERROR([mdblocales header not found.])]) +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/src/backend/utils/adt/Makefile b/src/backend/utils/adt/Makefile index 4a233b63c32..3e1878db6fb 100644 --- a/src/backend/utils/adt/Makefile +++ b/src/backend/utils/adt/Makefile @@ -124,7 +124,8 @@ OBJS = \ windowfuncs.o \ xid.o \ xid8funcs.o \ - xml.o + xml.o \ + mdb.o # See notes in src/backend/parser/Makefile about the following two rules jsonpath_gram.h: jsonpath_gram.c diff --git a/src/backend/utils/adt/mdb.c b/src/backend/utils/adt/mdb.c new file mode 100644 index 00000000000..cc61073fa58 --- /dev/null +++ b/src/backend/utils/adt/mdb.c @@ -0,0 +1,36 @@ +/*------------------------------------------------------------------------- + * + * mdb.c + * mdb routines + * + * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * + * IDENTIFICATION + * src/backend/utils/adt/mdb.c + * + *------------------------------------------------------------------------- + */ + + +#include "postgres.h" +#include "fmgr.h" + +/* + * mdb_admin_enabled + * Check that mdb locale patch is enabled + */ +Datum +mdb_locale_enabled(PG_FUNCTION_ARGS) +{ + bool res; + +#if USE_MDBLOCALES + res = true; +#else + res = false; +#endif + + PG_RETURN_BOOL(res); +} diff --git a/src/backend/utils/adt/pg_locale.c b/src/backend/utils/adt/pg_locale.c index cb1744f4d69..65ce9e131c0 100644 --- a/src/backend/utils/adt/pg_locale.c +++ b/src/backend/utils/adt/pg_locale.c @@ -48,6 +48,7 @@ #include "utils/pg_locale.h" #include "utils/relcache.h" #include "utils/syscache.h" +#include "common/mdb_locale.h" #ifdef WIN32 #include @@ -201,7 +202,7 @@ pg_perm_setlocale(int category, const char *locale) const char *envvar; #ifndef WIN32 - result = setlocale(category, locale); + result = SETLOCALE(category, locale); #else /* @@ -219,7 +220,7 @@ pg_perm_setlocale(int category, const char *locale) } else #endif - result = setlocale(category, locale); + result = SETLOCALE(category, locale); #endif /* WIN32 */ if (result == NULL) @@ -316,7 +317,7 @@ check_locale(int category, const char *locale, char **canonname) if (canonname) *canonname = NULL; /* in case of failure */ - save = setlocale(category, NULL); + save = SETLOCALE(category, NULL); if (!save) return false; /* won't happen, we hope */ @@ -324,14 +325,14 @@ check_locale(int category, const char *locale, char **canonname) save = pstrdup(save); /* set the locale with setlocale, to see if it accepts it. */ - res = setlocale(category, locale); + res = SETLOCALE(category, locale); /* save canonical name if requested. */ if (res && canonname) *canonname = pstrdup(res); /* restore old value. */ - if (!setlocale(category, save)) + if (!SETLOCALE(category, save)) elog(WARNING, "failed to restore old locale \"%s\"", save); pfree(save); @@ -749,7 +750,7 @@ cache_locale_time(void) if (locale == (locale_t) 0) _dosmaperr(GetLastError()); #else - locale = newlocale(LC_ALL_MASK, locale_time, (locale_t) 0); + locale = NEWLOCALE(LC_ALL_MASK, locale_time, (locale_t) 0); #endif if (!locale) report_newlocale_failure(locale_time); diff --git a/src/backend/utils/adt/pg_locale_libc.c b/src/backend/utils/adt/pg_locale_libc.c index c68fec2e585..3791544ac22 100644 --- a/src/backend/utils/adt/pg_locale_libc.c +++ b/src/backend/utils/adt/pg_locale_libc.c @@ -25,6 +25,8 @@ #include "utils/pg_locale.h" #include "utils/syscache.h" +#include "common/mdb_locale.h" + #ifdef __GLIBC__ #include #endif @@ -505,7 +507,7 @@ make_libc_collator(const char *collate, const char *ctype) /* Normal case where they're the same */ errno = 0; #ifndef WIN32 - loc = newlocale(LC_COLLATE_MASK | LC_CTYPE_MASK, collate, + loc = NEWLOCALE(LC_COLLATE_MASK | LC_CTYPE_MASK, collate, NULL); #else loc = _create_locale(LC_ALL, collate); @@ -523,7 +525,7 @@ make_libc_collator(const char *collate, const char *ctype) if (strcmp(collate, "C") != 0 && strcmp(collate, "POSIX") != 0) { errno = 0; - loc1 = newlocale(LC_COLLATE_MASK, collate, NULL); + loc1 = NEWLOCALE(LC_COLLATE_MASK, collate, NULL); if (!loc1) report_newlocale_failure(collate); } @@ -531,7 +533,7 @@ make_libc_collator(const char *collate, const char *ctype) if (strcmp(ctype, "C") != 0 && strcmp(ctype, "POSIX") != 0) { errno = 0; - loc = newlocale(LC_CTYPE_MASK, ctype, loc1); + loc = NEWLOCALE(LC_CTYPE_MASK, ctype, loc1); if (!loc) { if (loc1) @@ -671,7 +673,7 @@ get_collation_actual_version_libc(const char *collcollate) locale_t loc; /* Look up FreeBSD collation version. */ - loc = newlocale(LC_COLLATE_MASK, collcollate, NULL); + loc = NEWLOCALE(LC_COLLATE_MASK, collcollate, NULL); if (loc) { collversion = diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c index 576419dbca0..9cdd4ff23c6 100644 --- a/src/backend/utils/mb/mbutils.c +++ b/src/backend/utils/mb/mbutils.c @@ -42,6 +42,7 @@ #include "utils/memutils.h" #include "utils/relcache.h" #include "varatt.h" +#include "common/mdb_locale.h" /* * We maintain a simple linked list caching the fmgr lookup info for the @@ -1356,7 +1357,7 @@ pg_bind_textdomain_codeset(const char *domainname) int new_msgenc; #ifndef WIN32 - const char *ctype = setlocale(LC_CTYPE, NULL); + const char *ctype = SETLOCALE(LC_CTYPE, NULL); if (pg_strcasecmp(ctype, "C") == 0 || pg_strcasecmp(ctype, "POSIX") == 0) #endif diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c index f1a6db1a532..d2f4ce26acf 100644 --- a/src/bin/initdb/initdb.c +++ b/src/bin/initdb/initdb.c @@ -81,6 +81,7 @@ #include "getopt_long.h" #include "mb/pg_wchar.h" #include "miscadmin.h" +#include "common/mdb_locale.h" /* Ideally this would be in a .h file, but it hardly seems worth the trouble */ @@ -374,7 +375,8 @@ save_global_locale(int category) if (!save) pg_fatal("out of memory"); #else - save = setlocale(category, NULL); + save = SETLOCALE(LC_TIME, NULL); + if (!save) pg_fatal("setlocale() failed"); save = pg_strdup(save); @@ -392,7 +394,8 @@ restore_global_locale(int category, save_locale_t save) if (!_wsetlocale(category, save)) pg_fatal("failed to restore old locale"); #else - if (!setlocale(category, save)) + save = pg_strdup(save); + if (!SETLOCALE(category, save)) pg_fatal("failed to restore old locale \"%s\"", save); #endif free(save); @@ -2155,7 +2158,7 @@ locale_date_order(const char *locale) save = save_global_locale(LC_TIME); - setlocale(LC_TIME, locale); + SETLOCALE(LC_TIME, locale); memset(&testtime, 0, sizeof(testtime)); testtime.tm_mday = 22; @@ -2218,7 +2221,7 @@ check_locale_name(int category, const char *locale, char **canonname) locale = ""; /* set the locale with setlocale, to see if it accepts it. */ - res = setlocale(category, locale); + res = SETLOCALE(category, locale); /* save canonical name if requested. */ if (res && canonname) diff --git a/src/bin/pg_upgrade/check.c b/src/bin/pg_upgrade/check.c index 7fd4b3886da..b364e98237a 100644 --- a/src/bin/pg_upgrade/check.c +++ b/src/bin/pg_upgrade/check.c @@ -14,6 +14,8 @@ #include "fe_utils/string_utils.h" #include "pg_upgrade.h" #include "common/unicode_version.h" +#include "common/mdb_locale.h" + static void check_new_cluster_is_empty(void); static void check_is_install_user(ClusterInfo *cluster); diff --git a/src/common/exec.c b/src/common/exec.c index 8b690a10185..220d19d36b8 100644 --- a/src/common/exec.c +++ b/src/common/exec.c @@ -33,6 +33,8 @@ #include #include #include +#include "common/mdb_locale.h" + #ifdef EXEC_BACKEND #if defined(HAVE_SYS_PERSONALITY_H) @@ -434,7 +436,7 @@ set_pglocale_pgservice(const char *argv0, const char *app) /* don't set LC_ALL in the backend */ if (strcmp(app, PG_TEXTDOMAIN("postgres")) != 0) { - setlocale(LC_ALL, ""); + SETLOCALE(LC_ALL, ""); /* * One could make a case for reproducing here PostmasterMain()'s test diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index fb4f7f50350..e5cf6edc5ef 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -12481,7 +12481,6 @@ proname => 'brin_minmax_multi_summary_send', provolatile => 's', prorettype => 'bytea', proargtypes => 'pg_brin_minmax_multi_summary', prosrc => 'brin_minmax_multi_summary_send' }, - { oid => '6291', descr => 'arbitrary value from among input values', proname => 'any_value', prokind => 'a', proisstrict => 'f', prorettype => 'anyelement', proargtypes => 'anyelement', @@ -12489,7 +12488,6 @@ { oid => '6292', descr => 'aggregate transition function', proname => 'any_value_transfn', prorettype => 'anyelement', proargtypes => 'anyelement anyelement', prosrc => 'any_value_transfn' }, - { oid => '6321', descr => 'list of available WAL summary files', proname => 'pg_available_wal_summaries', prorows => '100', proretset => 't', provolatile => 'v', prorettype => 'record', proargtypes => '', @@ -12547,4 +12545,9 @@ proargnames => '{pid,io_id,io_generation,state,operation,off,length,target,handle_data_len,raw_result,result,target_desc,f_sync,f_localmem,f_buffered}', prosrc => 'pg_get_aios' }, +# MDB locale +{ oid => '16383', descr => 'contains', + proname => 'mdb_locale_enabled', prorettype => 'bool', + proargtypes => '', prosrc => 'mdb_locale_enabled' }, + ] diff --git a/src/include/common/mdb_locale.h b/src/include/common/mdb_locale.h new file mode 100644 index 00000000000..61290b2d938 --- /dev/null +++ b/src/include/common/mdb_locale.h @@ -0,0 +1,24 @@ +/*------------------------------------------------------------------------- + * + * locale_mdb.h + * Generic headers for custom MDB-locales patch. + * + * IDENTIFICATION + * src/include/common/mdb_locale.h + * + *------------------------------------------------------------------------- + */ + +#ifndef PG_MDB_LOCALE_H +#define PG_MDB_LOCALE_H + +#ifdef USE_MDBLOCALES +#include +#define SETLOCALE(category, locale) mdb_setlocale(category, locale) +#define NEWLOCALE(category, locale, base) mdb_newlocale(category, locale, base) +#else +#define SETLOCALE(category, locale) setlocale(category, locale) +#define NEWLOCALE(category, locale, base) newlocale(category, locale, base) +#endif + +#endif /* PG_MDB_LOCALE_H */ \ No newline at end of file diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 185f8bbea84..92191012241 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -629,6 +629,9 @@ /* A string containing the version number, platform, and C compiler */ #undef PG_VERSION_STR +/* Use mdb locales or not */ +#undef USE_MDBLOCALES + /* Define to 1 to allow profiling output to be saved separately for each process. */ #undef PROFILE_PID_DIR diff --git a/src/interfaces/ecpg/ecpglib/connect.c b/src/interfaces/ecpg/ecpglib/connect.c index 713cbbf6360..150259bbc1e 100644 --- a/src/interfaces/ecpg/ecpglib/connect.c +++ b/src/interfaces/ecpg/ecpglib/connect.c @@ -9,6 +9,7 @@ #include "ecpglib_extern.h" #include "ecpgtype.h" #include "sqlca.h" +#include "common/mdb_locale.h" #ifdef HAVE_USELOCALE locale_t ecpg_clocale = (locale_t) 0; @@ -495,7 +496,7 @@ ECPGconnect(int lineno, int c, const char *name, const char *user, const char *p #ifdef HAVE_USELOCALE if (!ecpg_clocale) { - ecpg_clocale = newlocale(LC_NUMERIC_MASK, "C", (locale_t) 0); + ecpg_clocale = NEWLOCALE(LC_NUMERIC_MASK, "C", (locale_t) 0); if (!ecpg_clocale) { pthread_mutex_unlock(&connections_mutex); diff --git a/src/interfaces/ecpg/ecpglib/descriptor.c b/src/interfaces/ecpg/ecpglib/descriptor.c index 651d5c8b2ed..089669f65f5 100644 --- a/src/interfaces/ecpg/ecpglib/descriptor.c +++ b/src/interfaces/ecpg/ecpglib/descriptor.c @@ -15,6 +15,8 @@ #include "sql3types.h" #include "sqlca.h" #include "sqlda.h" +#include "common/mdb_locale.h" + static void descriptor_free(struct descriptor *desc); @@ -493,8 +495,8 @@ ECPGget_desc(int lineno, const char *desc_name, int index,...) #ifdef WIN32 stmt.oldthreadlocale = _configthreadlocale(_ENABLE_PER_THREAD_LOCALE); #endif - stmt.oldlocale = ecpg_strdup(setlocale(LC_NUMERIC, NULL), lineno); - setlocale(LC_NUMERIC, "C"); + stmt.oldlocale = ecpg_strdup(SETLOCALE(LC_NUMERIC, NULL), lineno); + SETLOCALE(LC_NUMERIC, "C"); #endif /* desperate try to guess something sensible */ @@ -507,7 +509,7 @@ ECPGget_desc(int lineno, const char *desc_name, int index,...) #else if (stmt.oldlocale) { - setlocale(LC_NUMERIC, stmt.oldlocale); + SETLOCALE(LC_NUMERIC, stmt.oldlocale); ecpg_free(stmt.oldlocale); } #ifdef WIN32 diff --git a/src/interfaces/ecpg/ecpglib/execute.c b/src/interfaces/ecpg/ecpglib/execute.c index f52da06de9a..fde4bf58d05 100644 --- a/src/interfaces/ecpg/ecpglib/execute.c +++ b/src/interfaces/ecpg/ecpglib/execute.c @@ -31,6 +31,7 @@ #include "sqlca.h" #include "sqlda-compat.h" #include "sqlda-native.h" +#include "common/mdb_locale.h" /* * This function returns a newly malloced string that has ' and \ @@ -2001,13 +2002,13 @@ ecpg_do_prologue(int lineno, const int compat, const int force_indicator, return false; } #endif - stmt->oldlocale = ecpg_strdup(setlocale(LC_NUMERIC, NULL), lineno); + stmt->oldlocale = ecpg_strdup(SETLOCALE(LC_NUMERIC, NULL), lineno); if (stmt->oldlocale == NULL) { ecpg_do_epilogue(stmt); return false; } - setlocale(LC_NUMERIC, "C"); + SETLOCALE(LC_NUMERIC, "C"); #endif /* @@ -2222,7 +2223,7 @@ ecpg_do_epilogue(struct statement *stmt) #else if (stmt->oldlocale) { - setlocale(LC_NUMERIC, stmt->oldlocale); + SETLOCALE(LC_NUMERIC, stmt->oldlocale); #ifdef WIN32 _configthreadlocale(stmt->oldthreadlocale); #endif diff --git a/src/interfaces/libpq/Makefile b/src/interfaces/libpq/Makefile index da6650066d4..d7b46a92a9a 100644 --- a/src/interfaces/libpq/Makefile +++ b/src/interfaces/libpq/Makefile @@ -87,9 +87,9 @@ endif # that are built correctly for use in a shlib. SHLIB_LINK_INTERNAL = -lpgcommon_shlib -lpgport_shlib ifneq ($(PORTNAME), win32) -SHLIB_LINK += $(filter -lcrypt -ldes -lcom_err -lcrypto -lk5crypto -lkrb5 -lgssapi_krb5 -lgss -lgssapi -lssl -lsocket -lnsl -lresolv -lintl -ldl -lm, $(LIBS)) $(LDAP_LIBS_FE) $(PTHREAD_LIBS) +SHLIB_LINK += $(filter -lcrypt -ldes -lcom_err -lcrypto -lk5crypto -lkrb5 -lgssapi_krb5 -lgss -lgssapi -lssl -lsocket -lnsl -lresolv -lintl -ldl -lm -lmdblocales, $(LIBS)) $(LDAP_LIBS_FE) $(PTHREAD_LIBS) else -SHLIB_LINK += $(filter -lcrypt -ldes -lcom_err -lcrypto -lk5crypto -lkrb5 -lgssapi32 -lssl -lsocket -lnsl -lresolv -lintl -lm $(PTHREAD_LIBS), $(LIBS)) $(LDAP_LIBS_FE) +SHLIB_LINK += $(filter -lcrypt -ldes -lcom_err -lcrypto -lk5crypto -lkrb5 -lgssapi32 -lssl -lsocket -lnsl -lresolv -lintl -lm -lmdblocales $(PTHREAD_LIBS), $(LIBS)) $(LDAP_LIBS_FE) endif ifeq ($(PORTNAME), win32) SHLIB_LINK += -lshell32 -lws2_32 -lsecur32 $(filter -lcomerr32 -lkrb5_32, $(LIBS)) diff --git a/src/pl/plperl/plperl.c b/src/pl/plperl/plperl.c index 29cb4d7e47f..b2dd9f93db3 100644 --- a/src/pl/plperl/plperl.c +++ b/src/pl/plperl/plperl.c @@ -36,6 +36,7 @@ #include "utils/rel.h" #include "utils/syscache.h" #include "utils/typcache.h" +#include "common/mdb_locale.h" /* define our text domain for translations */ #undef TEXTDOMAIN @@ -742,15 +743,15 @@ plperl_init_interp(void) *save_numeric, *save_time; - loc = setlocale(LC_COLLATE, NULL); + loc = SETLOCALE(LC_COLLATE, NULL); save_collate = loc ? pstrdup(loc) : NULL; - loc = setlocale(LC_CTYPE, NULL); + loc = SETLOCALE(LC_CTYPE, NULL); save_ctype = loc ? pstrdup(loc) : NULL; - loc = setlocale(LC_MONETARY, NULL); + loc = SETLOCALE(LC_MONETARY, NULL); save_monetary = loc ? pstrdup(loc) : NULL; - loc = setlocale(LC_NUMERIC, NULL); + loc = SETLOCALE(LC_NUMERIC, NULL); save_numeric = loc ? pstrdup(loc) : NULL; - loc = setlocale(LC_TIME, NULL); + loc = SETLOCALE(LC_TIME, NULL); save_time = loc ? pstrdup(loc) : NULL; #define PLPERL_RESTORE_LOCALE(name, saved) \ @@ -4182,7 +4183,7 @@ static char * setlocale_perl(int category, char *locale) { dTHX; - char *RETVAL = setlocale(category, locale); + char *RETVAL = SETLOCALE(category, locale); if (RETVAL) { @@ -4197,7 +4198,7 @@ setlocale_perl(int category, char *locale) #ifdef LC_ALL if (category == LC_ALL) - newctype = setlocale(LC_CTYPE, NULL); + newctype = SETLOCALE(LC_CTYPE, NULL); else #endif newctype = RETVAL; @@ -4215,7 +4216,7 @@ setlocale_perl(int category, char *locale) #ifdef LC_ALL if (category == LC_ALL) - newcoll = setlocale(LC_COLLATE, NULL); + newcoll = SETLOCALE(LC_COLLATE, NULL); else #endif newcoll = RETVAL; @@ -4234,7 +4235,7 @@ setlocale_perl(int category, char *locale) #ifdef LC_ALL if (category == LC_ALL) - newnum = setlocale(LC_NUMERIC, NULL); + newnum = SETLOCALE(LC_NUMERIC, NULL); else #endif newnum = RETVAL; diff --git a/src/port/chklocale.c b/src/port/chklocale.c index 023a7435c9b..b6e71ec2f53 100644 --- a/src/port/chklocale.c +++ b/src/port/chklocale.c @@ -18,6 +18,8 @@ #else #include "postgres_fe.h" #endif +#include "common/mdb_locale.h" + #ifndef WIN32 #include @@ -309,7 +311,7 @@ pg_get_encoding_from_locale(const char *ctype, bool write_message) /* Get the CODESET property, and also LC_CTYPE if not passed in */ if (!ctype) - ctype = setlocale(LC_CTYPE, NULL); + ctype = SETLOCALE(LC_CTYPE, NULL); /* If locale is C or POSIX, we can allow all encodings */ @@ -319,7 +321,7 @@ pg_get_encoding_from_locale(const char *ctype, bool write_message) #ifndef WIN32 - loc = newlocale(LC_CTYPE_MASK, ctype, (locale_t) 0); + loc = NEWLOCALE(LC_CTYPE_MASK, ctype, (locale_t) 0); if (loc == (locale_t) 0) return -1; /* bogus ctype passed in? */ diff --git a/src/port/pg_localeconv_r.c b/src/port/pg_localeconv_r.c index 61510b2e0ea..7acb2b94ad3 100644 --- a/src/port/pg_localeconv_r.c +++ b/src/port/pg_localeconv_r.c @@ -22,6 +22,8 @@ #include +#include "common/mdb_locale.h" + #ifdef MON_THOUSANDS_SEP /* * One of glibc's extended langinfo items detected. Assume that the full set @@ -262,14 +264,14 @@ pg_localeconv_r(const char *lc_monetary, memset(output, 0, sizeof(*output)); /* Copy the LC_MONETARY members. */ - if (!setlocale(LC_ALL, lc_monetary)) + if (!SETLOCALE(LC_ALL, lc_monetary)) goto exit; result = pg_localeconv_copy_members(output, localeconv(), LC_MONETARY); if (result != 0) goto exit; /* Copy the LC_NUMERIC members. */ - if (!setlocale(LC_ALL, lc_numeric)) + if (!SETLOCALE(LC_ALL, lc_numeric)) goto exit; result = pg_localeconv_copy_members(output, localeconv(), LC_NUMERIC); @@ -306,10 +308,10 @@ pg_localeconv_r(const char *lc_monetary, * doesn't match. */ errno = ENOENT; - monetary_locale = newlocale(LC_ALL_MASK, lc_monetary, 0); + monetary_locale = NEWLOCALE(LC_ALL_MASK, lc_monetary, 0); if (monetary_locale == 0) return -1; - numeric_locale = newlocale(LC_ALL_MASK, lc_numeric, 0); + numeric_locale = NEWLOCALE(LC_ALL_MASK, lc_numeric, 0); if (numeric_locale == 0) { freelocale(monetary_locale); diff --git a/src/port/win32setlocale.c b/src/port/win32setlocale.c index 6e084946132..7fcbcb1f79b 100644 --- a/src/port/win32setlocale.c +++ b/src/port/win32setlocale.c @@ -180,7 +180,7 @@ pgwin32_setlocale(int category, const char *locale) argument = map_locale(locale_map_argument, locale); /* Call the real setlocale() function */ - result = setlocale(category, argument); + result = SETLOCALE(category, argument); /* * setlocale() is specified to return a "char *" that the caller is diff --git a/src/test/locale/test-ctype.c b/src/test/locale/test-ctype.c index a3f896c5ecb..10c2b49cb92 100644 --- a/src/test/locale/test-ctype.c +++ b/src/test/locale/test-ctype.c @@ -23,6 +23,8 @@ the author shall be liable for any damage, etc. #include #include #include +#include "common/mdb_locale.h" + char *flag(int b); void describe_char(int c); @@ -62,7 +64,7 @@ main() short c; char *cur_locale; - cur_locale = setlocale(LC_ALL, ""); + cur_locale = SETLOCALE(LC_ALL, ""); if (cur_locale) fprintf(stderr, "Successfully set locale to \"%s\"\n", cur_locale); else diff --git a/src/test/regress/expected/misc.out b/src/test/regress/expected/misc.out index 6e816c57f1f..d2cbc8f87f4 100644 --- a/src/test/regress/expected/misc.out +++ b/src/test/regress/expected/misc.out @@ -396,3 +396,10 @@ SELECT *, (equipment(CAST((h.*) AS hobbies_r))).name FROM hobbies_r h; -- -- rewrite rules -- +--- mdb-related +SELECT mdb_locale_enabled(); + mdb_locale_enabled +-------------------- + t +(1 row) + diff --git a/src/test/regress/sql/misc.sql b/src/test/regress/sql/misc.sql index 165a2e175fb..597186ffe4b 100644 --- a/src/test/regress/sql/misc.sql +++ b/src/test/regress/sql/misc.sql @@ -273,3 +273,8 @@ SELECT *, (equipment(CAST((h.*) AS hobbies_r))).name FROM hobbies_r h; -- -- rewrite rules -- + +--- mdb-related + +SELECT mdb_locale_enabled(); + From bdcd8061ce656c94585b70bc19ed6e841f186432 Mon Sep 17 00:00:00 2001 From: reshke kirill Date: Thu, 11 May 2023 08:36:59 +0000 Subject: [PATCH 19/45] MDB-23247: startup param for auth passthrough under unpriviledged user MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Accept new startup param _pq_.service_auth_role for service auth under any user, which is niether superuser nor have some dangerous system role priv Add tap-test for mdb service role auth 👍👌😉 Fix tests after rebase contrib tests 💅️️💅️️💅️️ now works MDB-23247: debug ouput for testing purposes lowered to DEBUG5 elog level Skip caching routines for pre-startup logic (SCRAM -service auth role) --- contrib/seg/Makefile | 2 + src/backend/libpq/auth.c | 26 +++++++- src/backend/tcop/backend_startup.c | 21 ++++--- src/backend/utils/adt/acl.c | 70 ++++++++++++++++----- src/bin/psql/t/100_mdb.pl | 98 +++++++++++++++++++++++++++++ src/include/libpq/libpq-be.h | 4 ++ src/include/utils/acl.h | 5 ++ src/interfaces/libpq/fe-connect.c | 8 +++ src/interfaces/libpq/fe-protocol3.c | 4 ++ src/interfaces/libpq/libpq-int.h | 1 + 10 files changed, 216 insertions(+), 23 deletions(-) create mode 100644 src/bin/psql/t/100_mdb.pl diff --git a/contrib/seg/Makefile b/contrib/seg/Makefile index ec7d4af7dc3..95673eec524 100644 --- a/contrib/seg/Makefile +++ b/contrib/seg/Makefile @@ -13,6 +13,8 @@ DATA = seg--1.1.sql seg--1.1--1.2.sql seg--1.2--1.3.sql seg--1.3--1.4.sql \ PGFILEDESC = "seg - line segment data type" HEADERS = segdata.h +# MDB-23261 diasble security test, because we do not extension creation via sql for now +#REGRESS = security seg REGRESS = seg partition diff --git a/src/backend/libpq/auth.c b/src/backend/libpq/auth.c index 4da46666439..d60b590e496 100644 --- a/src/backend/libpq/auth.c +++ b/src/backend/libpq/auth.c @@ -40,6 +40,8 @@ #include "storage/ipc.h" #include "tcop/backend_startup.h" #include "utils/memutils.h" +#include "utils/timestamp.h" +#include "utils/acl.h" /*---------------------------------------------------------------- * Global authentication functions @@ -825,12 +827,32 @@ CheckPWChallengeAuth(Port *port, const char **logdetail) int auth_result; char *shadow_pass; PasswordType pwtype; + Oid mdb_service_authoid; + Oid useroid; + Oid service_auth_roleoid; Assert(port->hba->auth_method == uaSCRAM || port->hba->auth_method == uaMD5); - /* First look up the user's password. */ - shadow_pass = get_role_password(port->user_name, logdetail); + + + if (port->service_auth_role) { + mdb_service_authoid = get_role_oid("mdb_service_auth", true); + service_auth_roleoid = get_role_oid(port->service_auth_role, true); + useroid = get_role_oid(port->user_name, true); + + /* MDB-23247: check that given role name has priviledge for auth - passthrough*/ + if (!is_member_of_role(service_auth_roleoid, mdb_service_authoid) || has_privs_of_unwanted_system_role_prestartup(useroid)) + ereport(FATAL, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("invalid auth request for role %s", + port->service_auth_role))); + + shadow_pass = get_role_password(port->service_auth_role, logdetail); + } else { + /* First look up the user's password. */ + shadow_pass = get_role_password(port->user_name, logdetail); + } /* * If the user does not exist, or has no password or it's expired, we diff --git a/src/backend/tcop/backend_startup.c b/src/backend/tcop/backend_startup.c index ad0af5edc1f..98902a98466 100644 --- a/src/backend/tcop/backend_startup.c +++ b/src/backend/tcop/backend_startup.c @@ -743,6 +743,8 @@ ProcessStartupPacket(Port *port, bool ssl_done, bool gss_done) break; /* missing value, will complain below */ valptr = buf + valoffset; + elog(DEBUG5, "startup got %s %s", nameptr, valptr); + if (strcmp(nameptr, "database") == 0) port->database_name = pstrdup(valptr); else if (strcmp(nameptr, "user") == 0) @@ -773,13 +775,18 @@ ProcessStartupPacket(Port *port, bool ssl_done, bool gss_done) } else if (strncmp(nameptr, "_pq_.", 5) == 0) { - /* - * Any option beginning with _pq_. is reserved for use as a - * protocol-level option, but at present no such options are - * defined. - */ - unrecognized_protocol_options = - lappend(unrecognized_protocol_options, pstrdup(nameptr)); + /* MDB-23247: parse service auth role from startup options */ + if (strcmp(nameptr, "_pq_.service_auth_role") == 0) { + port->service_auth_role = pstrdup(valptr); + } else { + /* + * Any option beginning with _pq_. is reserved for use as a + * protocol-level option, but at present no such options are + * defined. + */ + unrecognized_protocol_options = + lappend(unrecognized_protocol_options, pstrdup(nameptr)); + } } else { diff --git a/src/backend/utils/adt/acl.c b/src/backend/utils/adt/acl.c index f14086ad25a..076c6789a87 100644 --- a/src/backend/utils/adt/acl.c +++ b/src/backend/utils/adt/acl.c @@ -133,8 +133,6 @@ static AclResult pg_role_aclcheck(Oid role_oid, Oid roleid, AclMode mode); static void RoleMembershipCacheCallback(Datum arg, int cacheid, uint32 hashvalue); -static bool has_privs_of_unwanted_system_role(Oid role); - /* * Test whether an identifier char can be left unquoted in ACLs. @@ -5153,7 +5151,7 @@ roles_list_append(List *roles_list, bloom_filter **bf, Oid role) */ static List * roles_is_member_of(Oid roleid, enum RoleRecurseType type, - Oid admin_of, Oid *admin_role) + Oid admin_of, Oid *admin_role, bool no_cache) { Oid dba; List *roles_list; @@ -5170,7 +5168,6 @@ roles_is_member_of(Oid roleid, enum RoleRecurseType type, if (cached_role[type] == roleid && !OidIsValid(admin_of) && OidIsValid(cached_role[type])) return cached_roles[type]; - /* * Role expansion happens in a non-database backend when guc.c checks * ROLE_PG_READ_ALL_SETTINGS for a physical walsender SHOW command. In @@ -5268,7 +5265,10 @@ roles_is_member_of(Oid roleid, enum RoleRecurseType type, cached_role[type] = InvalidOid; /* just paranoia */ list_free(cached_roles[type]); cached_roles[type] = new_cached_roles; - cached_role[type] = roleid; + + if (!no_cache) { + cached_role[type] = roleid; + } /* And now we can return the answer */ return cached_roles[type]; @@ -5304,17 +5304,39 @@ has_privs_of_role_strict(Oid member, Oid role) * multi-level recursion, then see if target role is any one of them. */ return list_member_oid(roles_is_member_of(member, ROLERECURSE_PRIVS, - InvalidOid, NULL), + InvalidOid, NULL, false), + role); +} + + +static bool +has_privs_of_role_strict_no_cache(Oid member, Oid role) +{ + /* Fast path for simple case */ + if (member == role) + return true; + + /* Superusers have every privilege, so are part of every role */ + if (superuser_arg(member)) + return true; + + /* + * Find all the roles that member has the privileges of, including + * multi-level recursion, then see if target role is any one of them. + */ + return list_member_oid(roles_is_member_of(member, ROLERECURSE_PRIVS, + InvalidOid, NULL, true), role); } + /* * Check that role is either one of "dangerous" system role * or has "strict" (not through mdb_admin or mdb_superuser) * privs of this role */ -static bool +bool has_privs_of_unwanted_system_role(Oid role) { if (has_privs_of_role_strict(role, ROLE_PG_READ_SERVER_FILES)) { return true; @@ -5335,6 +5357,26 @@ has_privs_of_unwanted_system_role(Oid role) { return false; } +bool has_privs_of_unwanted_system_role_prestartup(Oid role) { + if (has_privs_of_role_strict_no_cache(role, ROLE_PG_READ_SERVER_FILES)) { + return true; + } + if (has_privs_of_role_strict_no_cache(role, ROLE_PG_WRITE_SERVER_FILES)) { + return true; + } + if (has_privs_of_role_strict_no_cache(role, ROLE_PG_EXECUTE_SERVER_PROGRAM)) { + return true; + } + if (has_privs_of_role_strict_no_cache(role, ROLE_PG_READ_ALL_DATA)) { + return true; + } + if (has_privs_of_role_strict_no_cache(role, ROLE_PG_WRITE_ALL_DATA)) { + return true; + } + + return false; +} + bool has_privs_of_role(Oid member, Oid role) { @@ -5370,7 +5412,7 @@ has_privs_of_role(Oid member, Oid role) * multi-level recursion, then see if target role is any one of them. */ return list_member_oid(roles_is_member_of(member, ROLERECURSE_PRIVS, - InvalidOid, NULL), + InvalidOid, NULL, false), role); } @@ -5447,7 +5489,7 @@ member_can_set_role(Oid member, Oid role) * multi-level recursion, then see if target role is any one of them. */ return list_member_oid(roles_is_member_of(member, ROLERECURSE_SETROLE, - InvalidOid, NULL), + InvalidOid, NULL, false), role); } @@ -5560,7 +5602,7 @@ is_member_of_role(Oid member, Oid role) * recursion, then see if target role is any one of them. */ return list_member_oid(roles_is_member_of(member, ROLERECURSE_MEMBERS, - InvalidOid, NULL), + InvalidOid, NULL, false), role); } @@ -5584,7 +5626,7 @@ is_member_of_role_nosuper(Oid member, Oid role) * recursion, then see if target role is any one of them. */ return list_member_oid(roles_is_member_of(member, ROLERECURSE_MEMBERS, - InvalidOid, NULL), + InvalidOid, NULL, false), role); } @@ -5606,7 +5648,7 @@ is_admin_of_role(Oid member, Oid role) if (member == role) return false; - (void) roles_is_member_of(member, ROLERECURSE_MEMBERS, role, &admin_role); + (void) roles_is_member_of(member, ROLERECURSE_MEMBERS, role, &admin_role, false); return OidIsValid(admin_role); } @@ -5628,7 +5670,7 @@ select_best_admin(Oid member, Oid role) if (member == role) return InvalidOid; - (void) roles_is_member_of(member, ROLERECURSE_PRIVS, role, &admin_role); + (void) roles_is_member_of(member, ROLERECURSE_PRIVS, role, &admin_role, false); return admin_role; } @@ -5697,7 +5739,7 @@ select_best_grantor(Oid roleId, AclMode privileges, * doesn't query any role memberships. */ roles_list = roles_is_member_of(roleId, ROLERECURSE_PRIVS, - InvalidOid, NULL); + InvalidOid, NULL, false); /* initialize candidate result as default */ *grantorId = roleId; *grantOptions = ACL_NO_RIGHTS; diff --git a/src/bin/psql/t/100_mdb.pl b/src/bin/psql/t/100_mdb.pl new file mode 100644 index 00000000000..d5b0342354c --- /dev/null +++ b/src/bin/psql/t/100_mdb.pl @@ -0,0 +1,98 @@ + +# Copyright (c) 2021-2023, PostgreSQL Global Development Group + +use strict; +use warnings; + +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +# Initialize primary node. Force UTF-8 encoding, so that we can use non-ASCII +# characters in the passwords below. +my $node = PostgreSQL::Test::Cluster->new('primary'); +$node->init(extra => [ '--locale=C', '--encoding=UTF8' ]); +$node->start; + + +# Create test roles. +$node->safe_psql( + 'postgres', + "SET password_encryption='scram-sha-256'; +SET client_encoding='utf8'; +CREATE USER mdbsar_test_role LOGIN PASSWORD 'test'; + +CREATE ROLE mdb_service_auth LOGIN PASSWORD 'serv'; +CREATE ROLE sup LOGIN SUPERUSER PASSWORD '123'; +"); + + +# Delete pg_hba.conf from the given node, add a new entry to it +# and then execute a reload to refresh it. +sub reset_pg_hba +{ + my $node = shift; + my $hba_method = shift; + + unlink($node->data_dir . '/pg_hba.conf'); + $node->append_conf('pg_hba.conf', "local all all $hba_method"); + $node->reload; + return; +} + +# Require password from now on. +reset_pg_hba($node, 'scram-sha-256'); + + + +# Test access for a single role, useful to wrap all tests into one. +sub test_login +{ + local $Test::Builder::Level = $Test::Builder::Level + 1; + + my $node = shift; + my $role = shift; + my $password = shift; + my $expected_res = shift; + my $add_serv_role = shift; + my $status_string = 'failed'; + + $status_string = 'success' if ($expected_res eq 0); + + my $connstr = "user=$role"; + my $testname = + "authentication $status_string for role $role with password $password"; + + $ENV{"PGPASSWORD"} = $password; + if ($add_serv_role eq 1) + { + $ENV{"PGSERVICEAUTHROLE"} = 'mdb_service_auth'; + } + + if ($expected_res eq 0) + { + $node->connect_ok($connstr, $testname); + } + else + { + # No checks of the error message, only the status code. + $node->connect_fails($connstr, $testname); + } +} + + +test_login($node, 'mdb_service_auth', "serv", 0, 0); +test_login($node, 'mdbsar_test_role', "serv", 2, 0); +test_login($node, 'mdbsar_test_role', "test", 0, 0); + +test_login($node, 'sup', "123", 0, 0); + +test_login($node, 'mdb_service_auth', "serv", 0, 1); +test_login($node, 'mdbsar_test_role', "serv", 0, 1); +test_login($node, 'mdbsar_test_role', "test", 2, 1); + +test_login($node, 'sup', "serv", 2, 1); + +ok(1); +done_testing(); + diff --git a/src/include/libpq/libpq-be.h b/src/include/libpq/libpq-be.h index d6e671a6382..db9702d8683 100644 --- a/src/include/libpq/libpq-be.h +++ b/src/include/libpq/libpq-be.h @@ -149,6 +149,10 @@ typedef struct Port */ char *database_name; char *user_name; + /* + * MDB-23247: service role name to perform auth - passthrough + */ + char *service_auth_role; char *cmdline_options; List *guc_options; diff --git a/src/include/utils/acl.h b/src/include/utils/acl.h index 7cfd92da9dc..913320706b4 100644 --- a/src/include/utils/acl.h +++ b/src/include/utils/acl.h @@ -213,6 +213,10 @@ extern bool has_privs_of_role(Oid member, Oid role); extern bool member_can_set_role(Oid member, Oid role); extern void check_can_set_role(Oid member, Oid role); extern bool is_member_of_role(Oid member, Oid role); + +// -- mdb patch +extern bool has_privs_of_unwanted_system_role_prestartup(Oid role); +// -- mdb patch end extern bool is_member_of_role_nosuper(Oid member, Oid role); extern bool is_admin_of_role(Oid member, Oid role); @@ -227,6 +231,7 @@ extern bool mdb_admin_is_member_of_role(Oid member, Oid role); extern Oid select_best_admin(Oid member, Oid role); extern Oid get_role_oid(const char *rolname, bool missing_ok); extern Oid get_role_oid_or_public(const char *rolname); +extern bool has_privs_of_unwanted_system_role(Oid role); extern Oid get_rolespec_oid(const RoleSpec *role, bool missing_ok); extern void check_rolespec_name(const RoleSpec *role, const char *detail_msg); extern HeapTuple get_rolespec_tuple(const RoleSpec *role); diff --git a/src/interfaces/libpq/fe-connect.c b/src/interfaces/libpq/fe-connect.c index e7770da3a58..dcbc8d7b55a 100644 --- a/src/interfaces/libpq/fe-connect.c +++ b/src/interfaces/libpq/fe-connect.c @@ -411,6 +411,11 @@ static const internalPQconninfoOption PQconninfoOptions[] = { {"sslkeylogfile", NULL, NULL, NULL, "SSL-Key-Log-File", "D", 64, offsetof(struct pg_conn, sslkeylogfile)}, + /* MDB-23247: option for service log-in */ + {"_pq_.service_auth_role", "PGSERVICEAUTHROLE", + "", NULL, + "_pg__service_auth_role", "", 20, + offsetof(struct pg_conn, service_auth_role)}, /* Terminating entry --- MUST BE LAST */ {NULL, NULL, NULL, NULL, @@ -5123,6 +5128,9 @@ freePGconn(PGconn *conn) free(conn->inBuffer); free(conn->outBuffer); free(conn->rowBuf); + if (conn->service_auth_role) { + free(conn->service_auth_role); + } termPQExpBuffer(&conn->errorMessage); termPQExpBuffer(&conn->workBuffer); diff --git a/src/interfaces/libpq/fe-protocol3.c b/src/interfaces/libpq/fe-protocol3.c index 38d01cbc024..08eac16cec2 100644 --- a/src/interfaces/libpq/fe-protocol3.c +++ b/src/interfaces/libpq/fe-protocol3.c @@ -2493,6 +2493,10 @@ build_startup_packet(const PGconn *conn, char *packet, } } + /* MDB-23247: add service auth role to startup options */ + if (conn->service_auth_role && conn->service_auth_role[0]) + ADD_STARTUP_OPTION("_pq_.service_auth_role", conn->service_auth_role); + /* Add trailing terminator */ if (packet) packet[packet_len] = '\0'; diff --git a/src/interfaces/libpq/libpq-int.h b/src/interfaces/libpq/libpq-int.h index decc09103aa..8ce598de2ce 100644 --- a/src/interfaces/libpq/libpq-int.h +++ b/src/interfaces/libpq/libpq-int.h @@ -432,6 +432,7 @@ struct pg_conn bool cancelRequest; /* true if this connection is used to send a * cancel request, instead of being a normal * connection that's used for queries */ + char *service_auth_role; /* MDB-23247: option for service log-in */ /* OAuth v2 */ char *oauth_issuer; /* token issuer/URL */ From eec08e1836b78313dcefbf1f05a4b690073971e4 Mon Sep 17 00:00:00 2001 From: reshke kirill Date: Wed, 5 Jul 2023 07:06:41 +0000 Subject: [PATCH 20/45] Add mdb changelog Update mdb-patched.md Update mdb-pacthes.md --- MDB-PATCHES.md | 90 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 MDB-PATCHES.md diff --git a/MDB-PATCHES.md b/MDB-PATCHES.md new file mode 100644 index 00000000000..a7d1efcbefc --- /dev/null +++ b/MDB-PATCHES.md @@ -0,0 +1,90 @@ + +15: + +7dfdf3a55ed: mdb_replication role patch +f6a3406fed8: Disallow cancelation of syncronous commit V1 +4289037edd2: Extend multixact SLRU +d6842b7f65c: Allow mdb_admin to create LEAKPROOF functions +146a82b3a75: mdb admin sets session replication role +68312eae50b: [MDB-16648]: Allow mdb admin to kill specific superuser queries +ab8f5243195: provide [mdb -postgresql] restict grant roles in YC[MDB-16990] +8ccf1aa7c51: Allow mdb admin to tranfers ownership on non-superuser objects regressioon tests for mdb admin functionality[MDB-16988] +bf4cad5ecdd: MDB-17910: check MDB reserved application name fix +64fed445a14: MDB-16955 : disallow to kill repl mon in cloud +afeb1eb4c2f: Fix mdb_replication role +5852300fb1d: pg_replication_slot_advance fix +c52bd070b56: Fix compilation errors + +0db90bbcbdd: Demonstrate and fix lock of all SQL queries by pg_stat_statements +530019f966d: MDB-21297: forbit usage of COPY TO PROGRAMM and COPY FROM PROGRAMM to non-superuser +8c860bf4d66: Reimplement mdb-admin, refactor mdb_admin check and usages. + +3a89cc36c74: Implement mdb-locales patch +dc7d503498b: Add mdb locales patch, restore COPY from/to files, enable regress. +96c30d707a7: Role mdb_superuser: feature and regress testsing +2d5f40ce3c9: Refactor optional setlocale, fix minor issues +41f04495a89: Update dependencies: bump libmdblocales, add mdb-locales +adc0b21d39f: Allow mdb_superuser to have power of pg_database_owner +ac90e1819fa: MDB-23247: startup param for auth passthrough under unpriviledged user +2bf6f042542: Add tap-test for mdb service role auth 👍👌😉 +9750b4efc44: Use fadvise to prefetch WAL in xlogrecovery +25f12802528: Fix tests after rebasecontrib tests 💅️️💅️️💅️️ now works +746dd65f557: MDB-23247: debug ouput for testing purposes lowered to DEBUG5 elog level + + + + + +16: + +/* misc */ + +/* on branch mdb-16 cherry-picked 'as is' */ +f6a3406fed8 -> 1effb23478e: Disallow cancelation of syncronous commit V1 +4289037edd2 -> b542d608604: Extend multixact SLRU + + + +/* mdb - admin + mdb_replication */ +7dfdf3a55ed: mdb_replication role patch + + +d6842b7f65c: Allow mdb_admin to create LEAKPROOF functions +146a82b3a75: mdb admin sets session replication role +68312eae50b: [MDB-16648]: Allow mdb admin to kill specific superuser queries +8ccf1aa7c51: Allow mdb admin to tranfers ownership on non-superuser objects regressioon tests for mdb admin functionality[MDB-16988] +8c860bf4d66: Reimplement mdb-admin, refactor mdb_admin check and usages. + +/* sqashed to */ +52435055d7b: Mdb-admin patch and regression tests +/*******/ + +/* as is */ +ab8f5243195->3fecc85426e: provide [mdb -postgresql] restict grant roles in YC[MDB-16990] + +/* pack of mdb patches */ +bf4cad5ecdd: MDB-17910: check MDB reserved application name fix +64fed445a14: MDB-16955 : disallow to kill repl mon in cloud +afeb1eb4c2f: Fix mdb_replication role +5852300fb1d: pg_replication_slot_advance fix +c52bd070b56: Fix compilation errors + +/* squashed to */ +52ea09c2d90: Pack of MDB-related patches: +/* */ + +0db90bbcbdd: Demonstrate and fix lock of all SQL queries by pg_stat_statements +530019f966d: MDB-21297: forbit usage of COPY TO PROGRAMM and COPY FROM PROGRAMM to non-superuser + +3a89cc36c74: Implement mdb-locales patch +dc7d503498b: Add mdb locales patch, restore COPY from/to files, enable regress. +96c30d707a7: Role mdb_superuser: feature and regress testsing +2d5f40ce3c9: Refactor optional setlocale, fix minor issues +41f04495a89: Update dependencies: bump libmdblocales, add mdb-locales +adc0b21d39f: Allow mdb_superuser to have power of pg_database_owner +ac90e1819fa: MDB-23247: startup param for auth passthrough under unpriviledged user +2bf6f042542: Add tap-test for mdb service role auth 👍👌😉 +9750b4efc44: Use fadvise to prefetch WAL in xlogrecovery +25f12802528: Fix tests after rebasecontrib tests 💅️️💅️️💅️️ now works +746dd65f557: MDB-23247: debug ouput for testing purposes lowered to DEBUG5 elog level + From 46228a8c2888256397b73c695ce74b6f69737c39 Mon Sep 17 00:00:00 2001 From: Kirill Reshke Date: Tue, 28 May 2024 14:18:25 +0300 Subject: [PATCH 21/45] Restrict DROP DATABASE to superuser only Add dropdb TAP test --- src/backend/commands/dbcommands.c | 9 +++++ src/test/mdb_admin/t/dropdb.pl | 55 +++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+) create mode 100644 src/test/mdb_admin/t/dropdb.pl diff --git a/src/backend/commands/dbcommands.c b/src/backend/commands/dbcommands.c index 5eb6caffe6d..22a367f0c2b 100644 --- a/src/backend/commands/dbcommands.c +++ b/src/backend/commands/dbcommands.c @@ -1721,6 +1721,15 @@ dropdb(const char *dbname, bool missing_ok, bool force) aclcheck_error(ACLCHECK_NOT_OWNER, OBJECT_DATABASE, dbname); + /* -- MDB kostyl begin + * remove this when feature will be supported + */ + if (!superuser()) + aclcheck_error(ACLCHECK_NO_PRIV, OBJECT_DATABASE, + dbname); + + /* MDB kostyl end */ + /* DROP hook for the database being removed */ InvokeObjectDropHook(DatabaseRelationId, db_id, 0); diff --git a/src/test/mdb_admin/t/dropdb.pl b/src/test/mdb_admin/t/dropdb.pl new file mode 100644 index 00000000000..c8b8902a94e --- /dev/null +++ b/src/test/mdb_admin/t/dropdb.pl @@ -0,0 +1,55 @@ + +# Copyright (c) 2024-2024, MDB, Mother Russia + +# Minimal test testing drop database restrictions +use strict; +use warnings; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +# Initialize primary node +my $node_primary = PostgreSQL::Test::Cluster->new('primary'); +$node_primary->init(); +$node_primary->start; + +# Create some content on primary and check its presence in standby nodes +$node_primary->safe_psql('postgres', + " + CREATE ROLE mdb_admin; + CREATE ROLE mdb_superuser; + CREATE ROLE mdb_reg_lh_1; + GRANT mdb_admin TO mdb_reg_lh_1; + GRANT mdb_superuser TO mdb_reg_lh_1; + ALTER ROLE mdb_reg_lh_1 CREATEDB; +"); + +# Create some content on primary +$node_primary->safe_psql('postgres', + " + SET ROLE mdb_reg_lh_1; + CREATE DATABASE regress_db1; +"); + + +my ($res_reg_lh_1, $stdout_reg_lh_1, $stderr_reg_lh_1) = $node_primary->psql('postgres', + " + SET ROLE mdb_reg_lh_1; + DROP DATABASE regress_db1; +"); + +# print ($res_reg_lh_1, $stdout_reg_lh_1, $stderr_reg_lh_1, "\n"); + +ok($res_reg_lh_1 != 0, "should fail for non-superuser"); +like($stderr_reg_lh_1, qr/ERROR: permission denied for database regress_db1/, "matches"); + +my ($res_reg_lh_2, $stdout_reg_lh_2, $stderr_reg_lh_2) = $node_primary->psql('postgres', + " + DROP DATABASE regress_db1; +"); + +ok($res_reg_lh_2 == 0, "should success for superuser"); + +# print ($res_reg_lh_2, $stdout_reg_lh_2, $stderr_reg_lh_2, "\n"); + +done_testing(); From 299e8501653eacd6b6dd2e638b8f92e47f47db93 Mon Sep 17 00:00:00 2001 From: Andrey Borodin Date: Mon, 8 Jul 2024 14:57:31 +0500 Subject: [PATCH 22/45] [MDB-28474] Increate readaheadchunk for XlogPageReader() We learned from the field incidents, that 128Kb is not enough. This time we are going to try to increase this value. --- src/backend/access/transam/xlogrecovery.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c index ff4d9edc790..8062d27f0c2 100644 --- a/src/backend/access/transam/xlogrecovery.c +++ b/src/backend/access/transam/xlogrecovery.c @@ -3461,6 +3461,18 @@ XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen, Assert(targetPageOff == readOff); Assert(reqLen <= readLen); +#if defined(USE_POSIX_FADVISE) && defined(POSIX_FADV_WILLNEED) + /* + * Prefetch next wal blocks to avoid page misses on next read iterations. + */ +#define RACHUNK (16*1024*1024) + if (readOff % RACHUNK == 0) { + pgstat_report_wait_start(WAIT_EVENT_WAL_PREFETCH); + posix_fadvise(readFile, readOff + RACHUNK, RACHUNK, POSIX_FADV_WILLNEED); + pgstat_report_wait_end(); + } +#endif + xlogreader->seg.ws_tli = curFileTLI; /* From fedd54f2d48e695982491e08118a73a4d92f67aa Mon Sep 17 00:00:00 2001 From: Jakub Wartak Date: Thu, 23 Jun 2022 08:18:26 +0000 Subject: [PATCH 23/45] Use fadvise to prefetch WAL in xlogrecovery Use fadvise in walsender remove bogus progress reporting Fix fadvise patch --- src/backend/access/transam/xlogreader.c | 10 +++++++++ src/backend/access/transam/xlogrecovery.c | 22 +++++++++---------- .../utils/activity/wait_event_names.txt | 1 + 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 5c26d33a603..7eec7990811 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -1574,6 +1574,16 @@ WALRead(XLogReaderState *state, /* Reset errno first; eases reporting non-errno-affecting errors */ errno = 0; + +#if defined(USE_POSIX_FADVISE) && defined(POSIX_FADV_WILLNEED) + /* + * Prefetch next wal blocks to avoid page misses on next read iterations. + */ +#define RACHUNK (16*1024*1024) + if (p == 0) + posix_fadvise(state->seg.ws_file, 0, RACHUNK, POSIX_FADV_WILLNEED); + +#endif readbytes = pg_pread(state->seg.ws_file, p, segbytes, (off_t) startoff); #ifndef FRONTEND diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c index 8062d27f0c2..5b7474141df 100644 --- a/src/backend/access/transam/xlogrecovery.c +++ b/src/backend/access/transam/xlogrecovery.c @@ -3423,6 +3423,16 @@ XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen, io_start = pgstat_prepare_io_time(track_wal_io_timing); pgstat_report_wait_start(WAIT_EVENT_WAL_READ); + +#if defined(USE_POSIX_FADVISE) && defined(POSIX_FADV_WILLNEED) + /* + * Prefetch next wal blocks to avoid page misses on next read iterations. + */ +#define RACHUNK (16*1024*1024) + if (readOff == 0) { + posix_fadvise(readFile, 0, RACHUNK, POSIX_FADV_WILLNEED); + } +#endif r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) readOff); if (r != XLOG_BLCKSZ) { @@ -3461,18 +3471,6 @@ XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen, Assert(targetPageOff == readOff); Assert(reqLen <= readLen); -#if defined(USE_POSIX_FADVISE) && defined(POSIX_FADV_WILLNEED) - /* - * Prefetch next wal blocks to avoid page misses on next read iterations. - */ -#define RACHUNK (16*1024*1024) - if (readOff % RACHUNK == 0) { - pgstat_report_wait_start(WAIT_EVENT_WAL_PREFETCH); - posix_fadvise(readFile, readOff + RACHUNK, RACHUNK, POSIX_FADV_WILLNEED); - pgstat_report_wait_end(); - } -#endif - xlogreader->seg.ws_tli = curFileTLI; /* diff --git a/src/backend/utils/activity/wait_event_names.txt b/src/backend/utils/activity/wait_event_names.txt index b9c1e6900ec..aea2f870731 100644 --- a/src/backend/utils/activity/wait_event_names.txt +++ b/src/backend/utils/activity/wait_event_names.txt @@ -268,6 +268,7 @@ WAL_COPY_WRITE "Waiting for a write when creating a new WAL segment by copying a WAL_INIT_SYNC "Waiting for a newly initialized WAL file to reach durable storage." WAL_INIT_WRITE "Waiting for a write while initializing a new WAL file." WAL_READ "Waiting for a read from a WAL file." +WAL_PREFETCH "Waiting for a WAL file prefetching." WAL_SUMMARY_READ "Waiting for a read from a WAL summary file." WAL_SUMMARY_WRITE "Waiting for a write to a WAL summary file." WAL_SYNC "Waiting for a WAL file to reach durable storage." From 4d5c86dac5ed1df4409ab6842db55520ff05189f Mon Sep 17 00:00:00 2001 From: diphantxm Date: Mon, 9 Sep 2024 16:50:15 +0300 Subject: [PATCH 24/45] parameter max_log_size to truncate logs There is no need to log the entire query, because it may be large and take lots of space on disk. Parameter max_log_size set the maximum length for logged query. Everything beyond that length is truncated. Value 0 disables the parameter. --- src/backend/utils/error/elog.c | 8 ++++++++ src/backend/utils/misc/guc_tables.c | 11 +++++++++++ src/backend/utils/misc/postgresql.conf.sample | 2 ++ src/bin/pg_ctl/t/004_logrotate.pl | 15 +++++++++++++++ src/include/utils/elog.h | 1 + 5 files changed, 37 insertions(+) diff --git a/src/backend/utils/error/elog.c b/src/backend/utils/error/elog.c index d78f3ab8a30..5345ecaa05f 100644 --- a/src/backend/utils/error/elog.c +++ b/src/backend/utils/error/elog.c @@ -112,6 +112,7 @@ int Log_destination = LOG_DESTINATION_STDERR; char *Log_destination_string = NULL; bool syslog_sequence_numbers = true; bool syslog_split_messages = true; +int max_log_size = 0; /* Processed form of backtrace_functions GUC */ static char *backtrace_function_list; @@ -1698,6 +1699,13 @@ EmitErrorReport(void) CHECK_STACK_DEPTH(); oldcontext = MemoryContextSwitchTo(edata->assoc_context); + if (max_log_size != 0 && debug_query_string != NULL) + { + char* str = debug_query_string; + str[pg_mbcliplen(str, strlen(str), max_log_size)] = '\0'; + debug_query_string = str; + } + /* * Reset the formatted timestamp fields before emitting any logs. This * includes all the log destinations and emit_log_hook, as the latter diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index 2155221f7a0..0356ebac4e0 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -3889,6 +3889,17 @@ struct config_int ConfigureNamesInt[] = NULL, NULL, NULL }, + { + {"max_log_size", PGC_SIGHUP, LOGGING_WHAT, + gettext_noop("Sets max size of logged statement."), + NULL + }, + &max_log_size, + 5 * (1024 * 1024), + 0, INT_MAX, + NULL, NULL, NULL + }, + /* End-of-list marker */ { {NULL, 0, 0, NULL, NULL}, NULL, 0, 0, 0, NULL, NULL, NULL diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index 5e75c106405..8df2b6fbb3b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -638,6 +638,8 @@ # bind-parameter values to N bytes; # -1 means print in full, 0 disables #log_statement = 'none' # none, ddl, mod, all +#max_log_size = 0 # max size of logged statement_timeout + # 0 disables the feature #log_replication_commands = off #log_temp_files = -1 # log temporary files equal or larger # than the specified size in kilobytes; diff --git a/src/bin/pg_ctl/t/004_logrotate.pl b/src/bin/pg_ctl/t/004_logrotate.pl index d78360e6d1a..c072b68c68a 100644 --- a/src/bin/pg_ctl/t/004_logrotate.pl +++ b/src/bin/pg_ctl/t/004_logrotate.pl @@ -69,6 +69,7 @@ sub check_log_pattern # these ensure stability of test results: log_rotation_age = 0 lc_messages = 'C' +max_log_size = 32 )); $node->start(); @@ -135,6 +136,20 @@ sub check_log_pattern check_log_pattern('csvlog', $new_current_logfiles, 'syntax error', $node); check_log_pattern('jsonlog', $new_current_logfiles, 'syntax error', $node); +$node->psql('postgres', 'INSERT INTO SOME_NON_EXISTANT_TABLE VALUES (TEST)'); +for (my $attempts = 0; $attempts < $max_attempts; $attempts++) +{ + eval { + $current_logfiles = slurp_file($node->data_dir . '/current_logfiles'); + }; + last unless $@; + usleep(100_000); +} +die $@ if $@; +check_log_pattern('stderr', $current_logfiles, 'INSERT INTO SOME_NON_EXISTANT_TA(?!(BLE VALUES \(TEST\)))', $node); +check_log_pattern('csvlog', $current_logfiles, 'INSERT INTO SOME_NON_EXISTANT_TA(?!(BLE VALUES \(TEST\)))', $node); +check_log_pattern('jsonlog', $current_logfiles, 'INSERT INTO SOME_NON_EXISTANT_TA(?!(BLE VALUES \(TEST\)))', $node); + $node->stop(); done_testing(); diff --git a/src/include/utils/elog.h b/src/include/utils/elog.h index 581b39091e0..0055935a669 100644 --- a/src/include/utils/elog.h +++ b/src/include/utils/elog.h @@ -493,6 +493,7 @@ extern PGDLLIMPORT int Log_destination; extern PGDLLIMPORT char *Log_destination_string; extern PGDLLIMPORT bool syslog_sequence_numbers; extern PGDLLIMPORT bool syslog_split_messages; +extern PGDLLIMPORT int max_log_size; /* Log destination bitmap */ #define LOG_DESTINATION_STDERR 1 From fee9ed1a5327045a2d63fc9599ac6e37d285ac29 Mon Sep 17 00:00:00 2001 From: Andrei Liarskii Date: Sat, 2 Nov 2024 08:39:52 +0000 Subject: [PATCH 25/45] Pull request #201: bump llvm to 18 17.0 Merge in MDB/postgres-dev from MDB-31374-bump-llvm-18-pg17_0 to MDB_17_0 Squashed commit of the following: commit 0931fd9d64288653cd950cd19d941bd4d51640d7 Author: Andrey Lyarskiy Date: Tue Oct 29 12:59:04 2024 +0300 bump llvm to 18 --- Dockerfile | 2 +- debian/control | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 709a8ea0218..1f457d2304c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,7 +30,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ libpam0g-dev \ debhelper debootstrap devscripts make equivs debhelper-compat \ libz-dev flex libicu-dev libio-pty-perl libipc-run-perl libkrb5-dev \ - libldap2-dev liblz4-dev liblz4-tool zstd libperl-dev libreadline-dev libselinux1-dev llvm-dev \ + libldap2-dev liblz4-dev liblz4-tool zstd libperl-dev libreadline-dev libselinux1-dev llvm-18-dev \ libsystemd-dev libxml2-dev libxml2-utils libxslt1-dev \ python3-dev systemtap-sdt-dev tcl-dev uuid-dev xsltproc zlib1g-dev \ bison dh-exec docbook-xml docbook-xsl \ diff --git a/debian/control b/debian/control index fde5f7d9c40..85dace476cd 100644 --- a/debian/control +++ b/debian/control @@ -11,7 +11,7 @@ Rules-Requires-Root: no Build-Depends: autoconf, bison, - clang [amd64 arm64 mips64el ppc64 ppc64el s390x] , + clang-18 [amd64 arm64 mips64el ppc64 ppc64el s390x] , debhelper-compat (= 13), dh-exec (>= 0.13~), docbook-xml, @@ -39,7 +39,7 @@ Build-Depends: libxml2-utils, libxslt1-dev, libzstd-dev (>= 1.4.0) , - llvm-dev [amd64 arm64 mips64el ppc64 ppc64el s390x] , + llvm-18-dev [amd64 arm64 mips64el ppc64 ppc64el s390x] , lz4 | liblz4-tool, mawk, perl (>= 5.8), From e7f92b94e8da9933edb9498443ce3d417c49bca6 Mon Sep 17 00:00:00 2001 From: diphantxm Date: Mon, 26 Aug 2024 16:17:08 +0300 Subject: [PATCH 26/45] Support FORCE option in analyze command Add new parameter to VACUUM command, FORCE, meaning VACUUM should terminate all backends that prevents the execution by holding conflicting lock Fix vacuum force Fix vacumm fixes for force vacuum Signed-off-by: roman khapov (cherry picked from commit 5a312d2bc32f9c47de40d51eaf41864e69cdf178) Signed-off-by: roman khapov Fix vacuumdb TAP --- doc/src/sgml/ref/analyze.sgml | 1 + doc/src/sgml/ref/vacuum.sgml | 11 ++++++ doc/src/sgml/ref/vacuumdb.sgml | 9 +++++ src/backend/commands/async.c | 28 +++++++++++++++ src/backend/commands/vacuum.c | 53 ++++++++++++++++++++++++++-- src/backend/storage/ipc/procsignal.c | 4 +++ src/backend/tcop/postgres.c | 5 +++ src/backend/utils/init/globals.c | 3 ++ src/bin/psql/tab-complete.in.c | 8 ++--- src/bin/scripts/t/100_vacuumdb.pl | 4 +++ src/bin/scripts/vacuumdb.c | 20 +++++++++++ src/include/commands/async.h | 2 ++ src/include/commands/vacuum.h | 1 + src/include/miscadmin.h | 3 ++ src/include/storage/procsignal.h | 3 ++ src/test/regress/expected/vacuum.out | 6 ++++ src/test/regress/sql/vacuum.sql | 6 ++++ 17 files changed, 161 insertions(+), 6 deletions(-) diff --git a/doc/src/sgml/ref/analyze.sgml b/doc/src/sgml/ref/analyze.sgml index ec81f00fecf..bf9e98a6c29 100644 --- a/doc/src/sgml/ref/analyze.sgml +++ b/doc/src/sgml/ref/analyze.sgml @@ -28,6 +28,7 @@ ANALYZE [ ( option [, ...] ) ] [ boolean ] SKIP_LOCKED [ boolean ] BUFFER_USAGE_LIMIT size + FORCE [ boolean ] and table_and_columns is: diff --git a/doc/src/sgml/ref/vacuum.sgml b/doc/src/sgml/ref/vacuum.sgml index bd5dcaf86a5..a9eeda3e6bc 100644 --- a/doc/src/sgml/ref/vacuum.sgml +++ b/doc/src/sgml/ref/vacuum.sgml @@ -34,6 +34,7 @@ VACUUM [ ( option [, ...] ) ] [ boolean ] PROCESS_TOAST [ boolean ] + FORCE [ boolean ] TRUNCATE [ boolean ] PARALLEL integer SKIP_DATABASE_STATS [ boolean ] @@ -184,6 +185,16 @@ VACUUM [ ( option [, ...] ) ] [ + + FORCE + + + Specifies that VACUUM should terminate all backends + holding conflicting lock. + + + + INDEX_CLEANUP diff --git a/doc/src/sgml/ref/vacuumdb.sgml b/doc/src/sgml/ref/vacuumdb.sgml index bfa3cf56096..d8eef30b85e 100644 --- a/doc/src/sgml/ref/vacuumdb.sgml +++ b/doc/src/sgml/ref/vacuumdb.sgml @@ -394,6 +394,15 @@ PostgreSQL documentation + + + + + Terminate backends holding conflicting lock + + + + diff --git a/src/backend/commands/async.c b/src/backend/commands/async.c index e1cf659485a..1f15ba821a7 100644 --- a/src/backend/commands/async.c +++ b/src/backend/commands/async.c @@ -1815,6 +1815,34 @@ HandleNotifyInterrupt(void) SetLatch(MyLatch); } + + +/* + * HandleRvrInterrupt + * + * Signal handler portion of interrupt handling. Let the backend know + * that there's a pending notify interrupt. If we're currently reading + * from the client, this will interrupt the read and + * ProcessClientReadInterrupt() will call ProcessNotifyInterrupt(). + */ +void +HandleRvrInterrupt(void) +{ + /* + * Note: this is called by a SIGNAL HANDLER. You must be very wary what + * you do here. + */ + + /* signal that work needs to be done */ + QueryCancelPending = true; + InterruptPending = true; + ProcDiePending = true; + QueryConflictForceVacuumPending = true; + + /* make sure the event is processed in due course */ + SetLatch(MyLatch); +} + /* * ProcessNotifyInterrupt * diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c index be863db81cb..471eb4882c1 100644 --- a/src/backend/commands/vacuum.c +++ b/src/backend/commands/vacuum.c @@ -165,6 +165,7 @@ ExecVacuum(ParseState *pstate, VacuumStmt *vacstmt, bool isTopLevel) BufferAccessStrategy bstrategy = NULL; bool verbose = false; bool skip_locked = false; + bool force = false; bool analyze = false; bool freeze = false; bool full = false; @@ -228,6 +229,8 @@ ExecVacuum(ParseState *pstate, VacuumStmt *vacstmt, bool isTopLevel) ring_size = result; } + else if (strcmp(opt->defname, "force") == 0) + force = defGetBoolean(opt); else if (!vacstmt->is_vacuumcmd) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), @@ -322,7 +325,8 @@ ExecVacuum(ParseState *pstate, VacuumStmt *vacstmt, bool isTopLevel) (process_main ? VACOPT_PROCESS_MAIN : 0) | (process_toast ? VACOPT_PROCESS_TOAST : 0) | (skip_database_stats ? VACOPT_SKIP_DATABASE_STATS : 0) | - (only_database_stats ? VACOPT_ONLY_DATABASE_STATS : 0); + (only_database_stats ? VACOPT_ONLY_DATABASE_STATS : 0) | + (force ? VACOPT_FORCE : 0); /* sanity checks on options */ Assert(params.options & (VACOPT_VACUUM | VACOPT_ANALYZE)); @@ -537,6 +541,15 @@ vacuum(List *relations, VacuumParams *params, BufferAccessStrategy bstrategy, errmsg("%s cannot be executed from VACUUM or ANALYZE", stmttype))); + /* sanity check for FORCE */ + if ((params->options & VACOPT_FORCE) != 0) + { + if ((params->options & VACOPT_SKIP_LOCKED) != 0) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("VACUUM option FORCE cannot be used with SKIP_LOCKED"))); + } + /* * Build list of relation(s) to process, putting any new data in * vac_context for safekeeping. @@ -806,6 +819,42 @@ vacuum_open_relation(Oid relid, RangeVar *relation, bits32 options, rel = try_relation_open(relid, lmode); else if (ConditionalLockRelationOid(relid, lmode)) rel = try_relation_open(relid, NoLock); + else if (options & VACOPT_FORCE) + { + LOCKTAG tag; + Oid dbid; + + if (IsSharedRelation(relid)) + dbid = InvalidOid; + else + dbid = MyDatabaseId; + + SET_LOCKTAG_RELATION(tag, dbid, relid); + + while (rel == NULL) + { + VirtualTransactionId* backends = GetLockConflicts(&tag, lmode, NULL); + + /* + * Send signals to all the backends holding the conflicting locks + */ + while (VirtualTransactionIdIsValid(*backends)) + { + SignalVirtualTransaction(*backends, + PROCSIG_CONFLICT_RVR_FORCE, + false); + backends++; + } + rel = try_relation_open(relid, lmode); + if (rel == NULL) + { + ereport(NOTICE, + (errcode(ERRCODE_LOCK_NOT_AVAILABLE), + errmsg("retrying attemts of acquiring lock for \"%s\" --- lock not available", + relation->relname))); + } + } + } else { rel = NULL; @@ -933,7 +982,7 @@ expand_vacuum_rel(VacuumRelation *vrel, MemoryContext vac_context, * below, as well as find_all_inheritors's expectation that the caller * holds some lock on the starting relation. */ - rvr_opts = (options & VACOPT_SKIP_LOCKED) ? RVR_SKIP_LOCKED : 0; + rvr_opts = ((options & VACOPT_SKIP_LOCKED) ? RVR_SKIP_LOCKED : 0); relid = RangeVarGetRelidExtended(vrel->relation, AccessShareLock, rvr_opts, diff --git a/src/backend/storage/ipc/procsignal.c b/src/backend/storage/ipc/procsignal.c index 05d99b452c3..fe891d738ac 100644 --- a/src/backend/storage/ipc/procsignal.c +++ b/src/backend/storage/ipc/procsignal.c @@ -719,6 +719,10 @@ procsignal_sigusr1_handler(SIGNAL_ARGS) if (CheckProcSignal(PROCSIG_RECOVERY_CONFLICT_BUFFERPIN)) HandleRecoveryConflictInterrupt(PROCSIG_RECOVERY_CONFLICT_BUFFERPIN); + /* MDB additions */ + if (CheckProcSignal(PROCSIG_CONFLICT_RVR_FORCE)) + HandleRvrInterrupt(); + SetLatch(MyLatch); } diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c index 115a634c641..9571af81c7b 100644 --- a/src/backend/tcop/postgres.c +++ b/src/backend/tcop/postgres.c @@ -3350,6 +3350,11 @@ ProcessInterrupts(void) proc_exit(0); } + else if (QueryConflictForceVacuumPending) { + ereport(FATAL, + (errcode(ERRCODE_ADMIN_SHUTDOWN), + errmsg("terminating connection due to conflict with force vacuum"))); + } else ereport(FATAL, (errcode(ERRCODE_ADMIN_SHUTDOWN), diff --git a/src/backend/utils/init/globals.c b/src/backend/utils/init/globals.c index d31cb45a058..aaa48890aba 100644 --- a/src/backend/utils/init/globals.c +++ b/src/backend/utils/init/globals.c @@ -44,6 +44,9 @@ volatile uint32 InterruptHoldoffCount = 0; volatile uint32 QueryCancelHoldoffCount = 0; volatile uint32 CritSectionCount = 0; +/* MDB additions */ +volatile sig_atomic_t QueryConflictForceVacuumPending = false; + int MyProcPid; pg_time_t MyStartTime; TimestampTz MyStartTimestamp; diff --git a/src/bin/psql/tab-complete.in.c b/src/bin/psql/tab-complete.in.c index fdd7e2308ca..737040f92bd 100644 --- a/src/bin/psql/tab-complete.in.c +++ b/src/bin/psql/tab-complete.in.c @@ -3112,8 +3112,8 @@ match_previous_words(int pattern_id, * one word, so the above test is correct. */ if (ends_with(prev_wd, '(') || ends_with(prev_wd, ',')) - COMPLETE_WITH("VERBOSE", "SKIP_LOCKED", "BUFFER_USAGE_LIMIT"); - else if (TailMatches("VERBOSE|SKIP_LOCKED")) + COMPLETE_WITH("VERBOSE", "SKIP_LOCKED", "BUFFER_USAGE_LIMIT", "FORCE"); + else if (TailMatches("VERBOSE|SKIP_LOCKED|FORCE")) COMPLETE_WITH("ON", "OFF"); } else if (Matches("ANALYZE", "(*)")) @@ -5185,8 +5185,8 @@ match_previous_words(int pattern_id, "DISABLE_PAGE_SKIPPING", "SKIP_LOCKED", "INDEX_CLEANUP", "PROCESS_MAIN", "PROCESS_TOAST", "TRUNCATE", "PARALLEL", "SKIP_DATABASE_STATS", - "ONLY_DATABASE_STATS", "BUFFER_USAGE_LIMIT"); - else if (TailMatches("FULL|FREEZE|ANALYZE|VERBOSE|DISABLE_PAGE_SKIPPING|SKIP_LOCKED|PROCESS_MAIN|PROCESS_TOAST|TRUNCATE|SKIP_DATABASE_STATS|ONLY_DATABASE_STATS")) + "ONLY_DATABASE_STATS", "BUFFER_USAGE_LIMIT", "FORCE"); + else if (TailMatches("FULL|FREEZE|ANALYZE|VERBOSE|DISABLE_PAGE_SKIPPING|SKIP_LOCKED|PROCESS_MAIN|PROCESS_TOAST|TRUNCATE|SKIP_DATABASE_STATS|ONLY_DATABASE_STATS|FORCE")) COMPLETE_WITH("ON", "OFF"); else if (TailMatches("INDEX_CLEANUP")) COMPLETE_WITH("AUTO", "ON", "OFF"); diff --git a/src/bin/scripts/t/100_vacuumdb.pl b/src/bin/scripts/t/100_vacuumdb.pl index ef2faf610b6..df76e9eac9f 100644 --- a/src/bin/scripts/t/100_vacuumdb.pl +++ b/src/bin/scripts/t/100_vacuumdb.pl @@ -48,6 +48,10 @@ [ 'vacuumdb', '--skip-locked', '--analyze-only', 'postgres' ], qr/statement: ANALYZE \(SKIP_LOCKED\).*;/, 'vacuumdb --skip-locked --analyze-only'); +$node->issues_sql_like( + [ 'vacuumdb', '--force', '--analyze-only', 'postgres' ], + qr/statement: ANALYZE \(FORCE\).*;/, + 'vacuumdb --force --analyze-only'); $node->command_fails( [ 'vacuumdb', '--analyze-only', '--disable-page-skipping', 'postgres' ], '--analyze-only and --disable-page-skipping specified together'); diff --git a/src/bin/scripts/vacuumdb.c b/src/bin/scripts/vacuumdb.c index e97ace18104..31c2b061c50 100644 --- a/src/bin/scripts/vacuumdb.c +++ b/src/bin/scripts/vacuumdb.c @@ -49,6 +49,7 @@ typedef struct vacuumingOptions bool skip_database_stats; char *buffer_usage_limit; bool missing_stats_only; + bool force; } vacuumingOptions; /* object filter options */ @@ -137,6 +138,7 @@ main(int argc, char *argv[]) {"no-process-main", no_argument, NULL, 12}, {"buffer-usage-limit", required_argument, NULL, 13}, {"missing-stats-only", no_argument, NULL, 14}, + {"force", no_argument, NULL, 15}, {NULL, 0, NULL, 0} }; @@ -287,6 +289,9 @@ main(int argc, char *argv[]) case 14: vacopts.missing_stats_only = true; break; + case 15: + vacopts.force = true; + break; default: /* getopt_long already emitted a complaint */ pg_log_error_hint("Try \"%s --help\" for more information.", progname); @@ -1162,6 +1167,13 @@ prepare_vacuum_command(PQExpBuffer sql, int serverVersion, appendPQExpBuffer(sql, "%sSKIP_LOCKED", sep); sep = comma; } + if (vacopts->force) + { + ///* FORCE is supported since v12 */ + Assert(serverVersion >= 120000); + appendPQExpBuffer(sql, "%sFORCE", sep); + sep = comma; + } if (vacopts->verbose) { appendPQExpBuffer(sql, "%sVERBOSE", sep); @@ -1248,6 +1260,13 @@ prepare_vacuum_command(PQExpBuffer sql, int serverVersion, appendPQExpBuffer(sql, "%sSKIP_LOCKED", sep); sep = comma; } + if (vacopts->force) + { + ///* FORCE is supported since v12 */ + Assert(serverVersion >= 120000); + appendPQExpBuffer(sql, "%sFORCE", sep); + sep = comma; + } if (vacopts->full) { appendPQExpBuffer(sql, "%sFULL", sep); @@ -1358,6 +1377,7 @@ help(const char *progname) printf(_(" -P, --parallel=PARALLEL_WORKERS use this many background workers for vacuum, if available\n")); printf(_(" -q, --quiet don't write any messages\n")); printf(_(" --skip-locked skip relations that cannot be immediately locked\n")); + printf(_(" --force terminate backends holding conflicting lock\n")); printf(_(" -t, --table='TABLE[(COLUMNS)]' vacuum specific table(s) only\n")); printf(_(" -v, --verbose write a lot of output\n")); printf(_(" -V, --version output version information, then exit\n")); diff --git a/src/include/commands/async.h b/src/include/commands/async.h index aaec7314c10..466e758fbf8 100644 --- a/src/include/commands/async.h +++ b/src/include/commands/async.h @@ -43,6 +43,8 @@ extern void AtPrepare_Notify(void); /* signal handler for inbound notifies (PROCSIG_NOTIFY_INTERRUPT) */ extern void HandleNotifyInterrupt(void); +extern void HandleRvrInterrupt(void); + /* process interrupts */ extern void ProcessNotifyInterrupt(bool flush); diff --git a/src/include/commands/vacuum.h b/src/include/commands/vacuum.h index bc37a80dc74..59b1d02e49a 100644 --- a/src/include/commands/vacuum.h +++ b/src/include/commands/vacuum.h @@ -188,6 +188,7 @@ typedef struct VacAttrStats #define VACOPT_DISABLE_PAGE_SKIPPING 0x100 /* don't skip any pages */ #define VACOPT_SKIP_DATABASE_STATS 0x200 /* skip vac_update_datfrozenxid() */ #define VACOPT_ONLY_DATABASE_STATS 0x400 /* only vac_update_datfrozenxid() */ +#define VACOPT_FORCE 0x800 /* terminate conflicting backend if cannot get lock */ /* * Values used by index_cleanup and truncate params. diff --git a/src/include/miscadmin.h b/src/include/miscadmin.h index 9a7d733ddef..52096c51a46 100644 --- a/src/include/miscadmin.h +++ b/src/include/miscadmin.h @@ -105,6 +105,9 @@ extern PGDLLIMPORT volatile uint32 InterruptHoldoffCount; extern PGDLLIMPORT volatile uint32 QueryCancelHoldoffCount; extern PGDLLIMPORT volatile uint32 CritSectionCount; +/* MDB additions */ +extern PGDLLIMPORT volatile sig_atomic_t QueryConflictForceVacuumPending; + /* in tcop/postgres.c */ extern void ProcessInterrupts(void); diff --git a/src/include/storage/procsignal.h b/src/include/storage/procsignal.h index 234cfcb364a..e57f227d770 100644 --- a/src/include/storage/procsignal.h +++ b/src/include/storage/procsignal.h @@ -37,6 +37,9 @@ typedef enum PROCSIG_LOG_MEMORY_CONTEXT, /* ask backend to log the memory contexts */ PROCSIG_PARALLEL_APPLY_MESSAGE, /* Message from parallel apply workers */ + /* MDB additions */ + PROCSIG_CONFLICT_RVR_FORCE, + /* Recovery conflict reasons */ PROCSIG_RECOVERY_CONFLICT_FIRST, PROCSIG_RECOVERY_CONFLICT_DATABASE = PROCSIG_RECOVERY_CONFLICT_FIRST, diff --git a/src/test/regress/expected/vacuum.out b/src/test/regress/expected/vacuum.out index 0abcc99989e..c71f89887ab 100644 --- a/src/test/regress/expected/vacuum.out +++ b/src/test/regress/expected/vacuum.out @@ -446,6 +446,12 @@ VACUUM (SKIP_LOCKED) vactst; VACUUM (SKIP_LOCKED, FULL) vactst; ANALYZE (SKIP_LOCKED) vactst; RESET client_min_messages; +-- FORCE option +ANALYZE (FORCE) vactst; +VACUUM (ANALYZE, FORCE) vactst; +VACUUM (FORCE) vactst; +VACUUM (ANALYZE, FORCE, SKIP_LOCKED) vactst; +ERROR: VACUUM option FORCE cannot be used with SKIP_LOCKED -- ensure VACUUM and ANALYZE don't have a problem with serializable SET default_transaction_isolation = serializable; VACUUM vactst; diff --git a/src/test/regress/sql/vacuum.sql b/src/test/regress/sql/vacuum.sql index a72bdb5b619..2f9f411f898 100644 --- a/src/test/regress/sql/vacuum.sql +++ b/src/test/regress/sql/vacuum.sql @@ -340,6 +340,12 @@ VACUUM (SKIP_LOCKED, FULL) vactst; ANALYZE (SKIP_LOCKED) vactst; RESET client_min_messages; +-- FORCE option +ANALYZE (FORCE) vactst; +VACUUM (ANALYZE, FORCE) vactst; +VACUUM (FORCE) vactst; +VACUUM (ANALYZE, FORCE, SKIP_LOCKED) vactst; + -- ensure VACUUM and ANALYZE don't have a problem with serializable SET default_transaction_isolation = serializable; VACUUM vactst; From bc7c1f7762d5122271f8f59cca90a91470ed0b95 Mon Sep 17 00:00:00 2001 From: diphantxm Date: Tue, 12 Nov 2024 15:45:28 +0300 Subject: [PATCH 27/45] truncate query to be logged in simple query --- src/backend/tcop/postgres.c | 10 ++++++++-- src/backend/utils/error/elog.c | 36 ++++++++++++++++++++++++++++------ src/include/utils/elog.h | 1 + 3 files changed, 39 insertions(+), 8 deletions(-) diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c index 9571af81c7b..e5e4e8850a0 100644 --- a/src/backend/tcop/postgres.c +++ b/src/backend/tcop/postgres.c @@ -72,6 +72,7 @@ #include "tcop/pquery.h" #include "tcop/tcopprot.h" #include "tcop/utility.h" +#include "utils/elog.h" #include "utils/guc_hooks.h" #include "utils/injection_point.h" #include "utils/lsyscache.h" @@ -1019,11 +1020,14 @@ exec_simple_query(const char *query_string) bool was_logged = false; bool use_implicit_block; char msec_str[32]; + char* query_log; /* * Report query to various monitoring facilities. */ debug_query_string = query_string; + bool copied = false; + query_log = build_query_log(query_string, &copied); pgstat_report_activity(STATE_RUNNING, query_string); @@ -1068,7 +1072,7 @@ exec_simple_query(const char *query_string) if (check_log_statement(parsetree_list)) { ereport(LOG, - (errmsg("statement: %s", query_string), + (errmsg("statement: %s", query_log), errhidestmt(true), errdetail_execute(parsetree_list))); was_logged = true; @@ -1367,7 +1371,7 @@ exec_simple_query(const char *query_string) case 2: ereport(LOG, (errmsg("duration: %s ms statement: %s", - msec_str, query_string), + msec_str, query_log), errhidestmt(true), errdetail_execute(parsetree_list))); break; @@ -1378,6 +1382,8 @@ exec_simple_query(const char *query_string) TRACE_POSTGRESQL_QUERY_DONE(query_string); + if (query_log && copied) + pfree(query_log); debug_query_string = NULL; } diff --git a/src/backend/utils/error/elog.c b/src/backend/utils/error/elog.c index 5345ecaa05f..0a3278d1349 100644 --- a/src/backend/utils/error/elog.c +++ b/src/backend/utils/error/elog.c @@ -1699,12 +1699,9 @@ EmitErrorReport(void) CHECK_STACK_DEPTH(); oldcontext = MemoryContextSwitchTo(edata->assoc_context); - if (max_log_size != 0 && debug_query_string != NULL) - { - char* str = debug_query_string; - str[pg_mbcliplen(str, strlen(str), max_log_size)] = '\0'; - debug_query_string = str; - } + const char* old_query_string = debug_query_string; + bool copied = false; + debug_query_string = build_query_log(debug_query_string, &copied); /* * Reset the formatted timestamp fields before emitting any logs. This @@ -1746,6 +1743,12 @@ EmitErrorReport(void) MemoryContextSwitchTo(oldcontext); recursion_depth--; + + if (debug_query_string && copied) + { + pfree(debug_query_string); + debug_query_string = old_query_string; + } } /* @@ -3832,3 +3835,24 @@ vwrite_stderr(const char *fmt, va_list ap) } #endif } + +char* +build_query_log(const char* query, bool *copied) +{ + *copied = false; + if (!query) + return NULL; + + size_t query_len = strlen(query); + if (max_log_size == 0 || query_len < max_log_size) + { + return query; + } + + *copied = true; + size_t query_log_len = pg_mbcliplen(query, query_len, max_log_size); + char* query_log = (char*)palloc(query_log_len+1); + memcpy(query_log, query, query_log_len); + query_log[query_log_len] = '\0'; + return query_log; +} diff --git a/src/include/utils/elog.h b/src/include/utils/elog.h index 0055935a669..6f3b3f84aa6 100644 --- a/src/include/utils/elog.h +++ b/src/include/utils/elog.h @@ -508,6 +508,7 @@ extern void log_status_format(StringInfo buf, const char *format, extern void DebugFileOpen(void); extern char *unpack_sql_state(int sql_state); extern bool in_error_recursion_trouble(void); +char* build_query_log(const char* query, bool *copied); /* Common functions shared across destinations */ extern void reset_formatted_start_time(void); From e507a8fb0dfa51e967031f6f7126fdca3609dfc6 Mon Sep 17 00:00:00 2001 From: reshke Date: Thu, 16 Jan 2025 19:19:02 +0300 Subject: [PATCH 28/45] Add CI Do not run mdb locales tests in OS Fix CI && fast CI circuit (#24) Use mirror apt repo (#35) Refactor regress Dockerfile Update Dockerfile Update entrypoint.sh Rewrite Dockerfile logic Co-authored-by: EinKrebs --- .github/workflows/regress.yml | 29 +++++++++++++++++++++ Dockerfile | 6 +++-- docker/regress/Dockerfile | 47 +++++++++++++++++++++++++++++++++++ docker/regress/run_tests.sh | 6 +++++ docker/regress/run_tests_f.sh | 6 +++++ 5 files changed, 92 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/regress.yml create mode 100644 docker/regress/Dockerfile create mode 100755 docker/regress/run_tests.sh create mode 100755 docker/regress/run_tests_f.sh diff --git a/.github/workflows/regress.yml b/.github/workflows/regress.yml new file mode 100644 index 00000000000..c8b48bcfb3c --- /dev/null +++ b/.github/workflows/regress.yml @@ -0,0 +1,29 @@ +name: Docker Image CI + +on: + push: + branches: [ "MDB_*" ] + pull_request: + branches: [ "MDB_*" ] + +jobs: + + check: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Build the Docker image + run: docker build . --file docker/regress/Dockerfile --tag regress_test:1234 && docker run --entrypoint /home/build-user/docker/regress/run_tests_f.sh regress_test:1234 + + check-world: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Build the Docker image + run: docker build . --file docker/regress/Dockerfile --tag regress_test:1234 && docker run --entrypoint /home/build-user/docker/regress/run_tests.sh regress_test:1234 + + diff --git a/Dockerfile b/Dockerfile index 1f457d2304c..871288a7ab8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,7 +15,9 @@ RUN sed -i 's/archive.ubuntu.com/mirror.yandex.ru/g' /etc/apt/sources.list &&\ apt-get update && apt-get install -y --no-install-recommends \ sudo build-essential \ gcc lsb-release libssl-dev gnupg openssl \ - gdb git curl + gdb git curl gnupg + +RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys FF5F4D0E27393420 RUN echo "deb http://dist.yandex.ru/mdb-${CODE_NAME}-secure stable/all/" >> /etc/apt/sources.list RUN echo "deb http://dist.yandex.ru/mdb-${CODE_NAME}-secure stable/\$(ARCH)/" >> /etc/apt/sources.list @@ -34,7 +36,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ libsystemd-dev libxml2-dev libxml2-utils libxslt1-dev \ python3-dev systemtap-sdt-dev tcl-dev uuid-dev xsltproc zlib1g-dev \ bison dh-exec docbook-xml docbook-xsl \ - clang libcurl4-openssl-dev libnuma-dev liburing-dev libzstd-dev pkgconf tzdata + clang-18 libcurl4-openssl-dev libnuma-dev liburing-dev libzstd-dev pkgconf tzdata RUN apt-get install -y \ libmdblocales1 libmdblocales-dev \ postgresql-client-common=${PGDG_VER} \ diff --git a/docker/regress/Dockerfile b/docker/regress/Dockerfile new file mode 100644 index 00000000000..19dae6e983d --- /dev/null +++ b/docker/regress/Dockerfile @@ -0,0 +1,47 @@ +ARG codename +FROM ubuntu:${codename:-bionic} + +ARG codename +ENV CODE_NAME=${codename:-bionic} + +ENV DEBIAN_FRONTEND=noninteractive +ENV TZ=Europe/Moskow +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + +RUN apt update && apt install -y ca-certificates +RUN sed -i 's/archive.ubuntu.com/mirror.yandex.ru/g' /etc/apt/sources.list + +RUN apt-get update && apt-get install -y --no-install-recommends \ + sudo build-essential \ + gcc lsb-release libssl-dev gnupg openssl \ + gdb git curl ca-certificates + +RUN curl -s 'http://keyserver.ubuntu.com/pks/lookup?op=get&search=0xafc3ce0d00e3c45a357e9e637fcd11186050cd1a' | \ + gpg --dearmour -o /etc/apt/trusted.gpg.d/yandex.gpg + +RUN apt-get update && apt-get install -y --no-install-recommends \ + sudo build-essential \ + gcc lsb-release libssl-dev gnupg openssl \ + gdb git \ + libpam0g-dev \ + debhelper debootstrap devscripts make equivs debhelper-compat \ + libz-dev flex libicu-dev libio-pty-perl libipc-run-perl libkrb5-dev \ + libldap2-dev liblz4-dev liblz4-tool zstd libperl-dev libreadline-dev libselinux1-dev llvm-dev \ + libsystemd-dev libxml2-dev libxml2-utils libxslt1-dev \ + pkg-config python3-dev systemtap-sdt-dev tcl-dev uuid-dev xsltproc zlib1g-dev \ + bison dh-exec docbook-xml docbook-xsl + +RUN groupadd -g 999 build-user && \ + useradd -r -u 999 -g build-user build-user + +COPY . /home/build-user +RUN chown build-user:build-user /home -R && usermod -aG sudo build-user + +RUN echo 'build-user ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers + +USER build-user +WORKDIR /home/build-user + +RUN CFLAGS="" ./configure --prefix=/home/build-user/pgbin --without-mdblocales --enable-depend --enable-cassert --enable-debug --enable-tap-tests && \ + make -j8 && \ + sudo make install diff --git a/docker/regress/run_tests.sh b/docker/regress/run_tests.sh new file mode 100755 index 00000000000..5b72eaea60d --- /dev/null +++ b/docker/regress/run_tests.sh @@ -0,0 +1,6 @@ +#!/bin/bash +set -ex + +sed -i '/mdb-related/,$d' src/test/regress/*/misc.* + +make check-world diff --git a/docker/regress/run_tests_f.sh b/docker/regress/run_tests_f.sh new file mode 100755 index 00000000000..a7ff7f136aa --- /dev/null +++ b/docker/regress/run_tests_f.sh @@ -0,0 +1,6 @@ +#!/bin/bash +set -ex + +sed -i '/mdb-related/,$d' src/test/regress/*/misc.* + +make check From 4a64440797ee1087238ba5d10cfa16d539b8d91f Mon Sep 17 00:00:00 2001 From: reshke Date: Fri, 17 Jan 2025 14:21:36 +0500 Subject: [PATCH 29/45] MDB-32132: fix grantor selection for mdb_superuser (#6) --- src/backend/utils/adt/acl.c | 2 +- src/test/regress/expected/mdb_superuser.out | 21 +++++++++++++++++ src/test/regress/sql/mdb_superuser.sql | 26 ++++++++++++++++++++- 3 files changed, 47 insertions(+), 2 deletions(-) diff --git a/src/backend/utils/adt/acl.c b/src/backend/utils/adt/acl.c index 076c6789a87..4271a4152ea 100644 --- a/src/backend/utils/adt/acl.c +++ b/src/backend/utils/adt/acl.c @@ -5726,7 +5726,7 @@ select_best_grantor(Oid roleId, AclMode privileges, if (is_member_of_role(GetUserId(), mdb_superuser_roleoid) && has_privs_of_role(GetUserId(), ownerId)) { - *grantorId = mdb_superuser_roleoid; + *grantorId = ownerId; AclMode mdb_superuser_allowed_privs = needed_goptions; *grantOptions = mdb_superuser_allowed_privs; return; diff --git a/src/test/regress/expected/mdb_superuser.out b/src/test/regress/expected/mdb_superuser.out index e8dbe2fe1cc..8d989661434 100644 --- a/src/test/regress/expected/mdb_superuser.out +++ b/src/test/regress/expected/mdb_superuser.out @@ -1,11 +1,15 @@ CREATE ROLE regress_mdb_superuser_user1; CREATE ROLE regress_mdb_superuser_user2; CREATE ROLE regress_mdb_superuser_user3; +CREATE ROLE regress_mdb_su_role_o1; +CREATE ROLE regress_mdb_su_role_o2; +GRANT mdb_superuser TO regress_mdb_su_role_o1; GRANT mdb_admin TO mdb_superuser; CREATE ROLE regress_superuser WITH SUPERUSER; GRANT mdb_superuser TO regress_mdb_superuser_user1; GRANT CREATE ON DATABASE regression TO regress_mdb_superuser_user2; GRANT CREATE ON DATABASE regression TO regress_mdb_superuser_user3; +GRANT CREATE ON DATABASE regression TO regress_mdb_su_role_o2; SET ROLE regress_mdb_superuser_user2; CREATE FUNCTION regress_mdb_superuser_add(integer, integer) RETURNS integer AS 'SELECT $1 + $2;' @@ -90,6 +94,20 @@ ERROR: permission denied for schema regtest SET ROLE regress_mdb_superuser_user1; GRANT ALL ON TABLE regtest.regtest TO regress_mdb_superuser_user1; ALTER TABLE regtest.regtest OWNER TO regress_mdb_superuser_user1; +-- Check grantor +SET ROLE regress_mdb_su_role_o2; +CREATE TABLE public.role_o2_t(); +SET ROLE mdb_superuser; +GRANT SELECT ON public.role_o2_t TO regress_mdb_su_role_o1; +SELECT + grantor +from information_schema.role_table_grants +where grantee='regress_mdb_su_role_o1' AND table_name = 'role_o2_t'; + grantor +------------------------ + regress_mdb_su_role_o2 +(1 row) + \c regression DROP DATABASE regress_check_owner; -- end tests @@ -97,6 +115,9 @@ RESET SESSION AUTHORIZATION; -- REVOKE CREATE ON DATABASE regression FROM regress_mdb_superuser_user2; REVOKE CREATE ON DATABASE regression FROM regress_mdb_superuser_user3; +REVOKE CREATE ON DATABASE regression FROM regress_mdb_su_role_o2; +DROP ROLE regress_mdb_su_role_o1; +DROP ROLE regress_mdb_su_role_o2; DROP VIEW regress_mdb_superuser_view; DROP FUNCTION regress_mdb_superuser_add; DROP TABLE regress_mdb_superuser_schema.regress_mdb_superuser_table; diff --git a/src/test/regress/sql/mdb_superuser.sql b/src/test/regress/sql/mdb_superuser.sql index 77dc489f715..b0e702c630c 100644 --- a/src/test/regress/sql/mdb_superuser.sql +++ b/src/test/regress/sql/mdb_superuser.sql @@ -2,6 +2,11 @@ CREATE ROLE regress_mdb_superuser_user1; CREATE ROLE regress_mdb_superuser_user2; CREATE ROLE regress_mdb_superuser_user3; +CREATE ROLE regress_mdb_su_role_o1; +CREATE ROLE regress_mdb_su_role_o2; + +GRANT mdb_superuser TO regress_mdb_su_role_o1; + GRANT mdb_admin TO mdb_superuser; CREATE ROLE regress_superuser WITH SUPERUSER; @@ -10,7 +15,7 @@ GRANT mdb_superuser TO regress_mdb_superuser_user1; GRANT CREATE ON DATABASE regression TO regress_mdb_superuser_user2; GRANT CREATE ON DATABASE regression TO regress_mdb_superuser_user3; - +GRANT CREATE ON DATABASE regression TO regress_mdb_su_role_o2; SET ROLE regress_mdb_superuser_user2; @@ -118,6 +123,21 @@ SET ROLE regress_mdb_superuser_user1; GRANT ALL ON TABLE regtest.regtest TO regress_mdb_superuser_user1; ALTER TABLE regtest.regtest OWNER TO regress_mdb_superuser_user1; +-- Check grantor + +SET ROLE regress_mdb_su_role_o2; + +CREATE TABLE public.role_o2_t(); + +SET ROLE mdb_superuser; + +GRANT SELECT ON public.role_o2_t TO regress_mdb_su_role_o1; + +SELECT + grantor +from information_schema.role_table_grants +where grantee='regress_mdb_su_role_o1' AND table_name = 'role_o2_t'; + \c regression DROP DATABASE regress_check_owner; @@ -127,6 +147,10 @@ RESET SESSION AUTHORIZATION; -- REVOKE CREATE ON DATABASE regression FROM regress_mdb_superuser_user2; REVOKE CREATE ON DATABASE regression FROM regress_mdb_superuser_user3; +REVOKE CREATE ON DATABASE regression FROM regress_mdb_su_role_o2; + +DROP ROLE regress_mdb_su_role_o1; +DROP ROLE regress_mdb_su_role_o2; DROP VIEW regress_mdb_superuser_view; DROP FUNCTION regress_mdb_superuser_add; From 4ad9183c45e24d46f293d20d5caa733d85af06dd Mon Sep 17 00:00:00 2001 From: reshke Date: Tue, 21 Jan 2025 16:14:43 +0500 Subject: [PATCH 30/45] Do not use schema public in mdb_superuser regression tests (#7) --- src/test/regress/expected/mdb_superuser.out | 5 +++-- src/test/regress/sql/mdb_superuser.sql | 6 ++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/test/regress/expected/mdb_superuser.out b/src/test/regress/expected/mdb_superuser.out index 8d989661434..7849311cd1d 100644 --- a/src/test/regress/expected/mdb_superuser.out +++ b/src/test/regress/expected/mdb_superuser.out @@ -85,6 +85,7 @@ CREATE DATABASE regress_check_owner OWNER regress_mdb_superuser_user2; SET ROLE regress_mdb_superuser_user2; CREATE SCHEMA regtest; CREATE TABLE regtest.regtest(); +GRANT CREATE ON SCHEMA regtest TO regress_mdb_su_role_o2; -- this should fail SET ROLE regress_mdb_superuser_user3; GRANT ALL ON TABLE regtest.regtest TO regress_mdb_superuser_user3; @@ -96,9 +97,9 @@ GRANT ALL ON TABLE regtest.regtest TO regress_mdb_superuser_user1; ALTER TABLE regtest.regtest OWNER TO regress_mdb_superuser_user1; -- Check grantor SET ROLE regress_mdb_su_role_o2; -CREATE TABLE public.role_o2_t(); +CREATE TABLE regtest.role_o2_t(); SET ROLE mdb_superuser; -GRANT SELECT ON public.role_o2_t TO regress_mdb_su_role_o1; +GRANT SELECT ON regtest.role_o2_t TO regress_mdb_su_role_o1; SELECT grantor from information_schema.role_table_grants diff --git a/src/test/regress/sql/mdb_superuser.sql b/src/test/regress/sql/mdb_superuser.sql index b0e702c630c..0877d9258f6 100644 --- a/src/test/regress/sql/mdb_superuser.sql +++ b/src/test/regress/sql/mdb_superuser.sql @@ -113,6 +113,8 @@ SET ROLE regress_mdb_superuser_user2; CREATE SCHEMA regtest; CREATE TABLE regtest.regtest(); +GRANT CREATE ON SCHEMA regtest TO regress_mdb_su_role_o2; + -- this should fail SET ROLE regress_mdb_superuser_user3; @@ -127,11 +129,11 @@ ALTER TABLE regtest.regtest OWNER TO regress_mdb_superuser_user1; SET ROLE regress_mdb_su_role_o2; -CREATE TABLE public.role_o2_t(); +CREATE TABLE regtest.role_o2_t(); SET ROLE mdb_superuser; -GRANT SELECT ON public.role_o2_t TO regress_mdb_su_role_o1; +GRANT SELECT ON regtest.role_o2_t TO regress_mdb_su_role_o1; SELECT grantor From 8ae34cab92704c34d5261fd7caa521457f22a3a5 Mon Sep 17 00:00:00 2001 From: reshke Date: Fri, 7 Feb 2025 13:07:37 +0000 Subject: [PATCH 31/45] Introduce mdb_read_all_data/mdb_write_all_data --- src/backend/catalog/aclchk.c | 29 ++++++ .../regress/expected/mdb_read_write_roles.out | 82 +++++++++++++++++ src/test/regress/expected/mdb_superuser.out | 4 +- src/test/regress/expected/test_setup.out | 3 + src/test/regress/parallel_schedule | 2 + src/test/regress/sql/mdb_read_write_roles.sql | 89 +++++++++++++++++++ src/test/regress/sql/mdb_superuser.sql | 5 +- src/test/regress/sql/test_setup.sql | 3 + 8 files changed, 212 insertions(+), 5 deletions(-) create mode 100644 src/test/regress/expected/mdb_read_write_roles.out create mode 100644 src/test/regress/sql/mdb_read_write_roles.sql diff --git a/src/backend/catalog/aclchk.c b/src/backend/catalog/aclchk.c index 17816481760..20049ad7617 100644 --- a/src/backend/catalog/aclchk.c +++ b/src/backend/catalog/aclchk.c @@ -3290,6 +3290,8 @@ pg_class_aclmask_ext(Oid table_oid, Oid roleid, AclMode mask, bool isNull; Acl *acl; Oid ownerId; + Oid mdb_read_all_data_oid; + Oid mdb_write_all_data_oid; /* * Must get the relation's tuple from pg_class @@ -3340,6 +3342,9 @@ pg_class_aclmask_ext(Oid table_oid, Oid roleid, AclMode mask, */ ownerId = classForm->relowner; + mdb_read_all_data_oid = get_role_oid("mdb_read_all_data", true); + mdb_write_all_data_oid = get_role_oid("mdb_write_all_data", true); + aclDatum = SysCacheGetAttr(RELOID, tuple, Anum_pg_class_relacl, &isNull); if (isNull) @@ -3379,6 +3384,17 @@ pg_class_aclmask_ext(Oid table_oid, Oid roleid, AclMode mask, has_privs_of_role(roleid, ROLE_PG_READ_ALL_DATA)) result |= ACL_SELECT; + + /* + * Check if ACL_SELECT is being checked and, if so, and not set already as + * part of the result, then check if the user is a member of the + * mdb_read_all_data role, and this is not some dangerous relation to grant SELECT to + */ + if (mask & ACL_SELECT && !(result & ACL_SELECT) && + has_privs_of_role(roleid, mdb_read_all_data_oid) && + !has_privs_of_unwanted_system_role(ownerId)) + result |= ACL_SELECT; + /* * Check if ACL_INSERT, ACL_UPDATE, or ACL_DELETE is being checked and, if * so, and not set already as part of the result, then check if the user @@ -3391,6 +3407,19 @@ pg_class_aclmask_ext(Oid table_oid, Oid roleid, AclMode mask, has_privs_of_role(roleid, ROLE_PG_WRITE_ALL_DATA)) result |= (mask & (ACL_INSERT | ACL_UPDATE | ACL_DELETE)); + + /* + * Check if ACL_INSERT, ACL_UPDATE, or ACL_DELETE is being checked and, if + * so, and not set already as part of the result, then check if the user + * is a member of the mdb_write_all_data role, and this is not some + * dangerous relation to grant write access. + */ + if (mask & (ACL_INSERT | ACL_UPDATE | ACL_DELETE) && + !(result & (ACL_INSERT | ACL_UPDATE | ACL_DELETE)) && + has_privs_of_role(roleid, mdb_write_all_data_oid) && + !has_privs_of_unwanted_system_role(ownerId)) + result |= (mask & (ACL_INSERT | ACL_UPDATE | ACL_DELETE)); + /* * Check if ACL_MAINTAIN is being checked and, if so, and not already set * as part of the result, then check if the user is a member of the diff --git a/src/test/regress/expected/mdb_read_write_roles.out b/src/test/regress/expected/mdb_read_write_roles.out new file mode 100644 index 00000000000..080e95cafb9 --- /dev/null +++ b/src/test/regress/expected/mdb_read_write_roles.out @@ -0,0 +1,82 @@ +CREATE ROLE regress_mdb_superuser_user1; +CREATE ROLE regress_mdb_superuser_user2; +CREATE ROLE regress_mdb_superuser_user3; +CREATE ROLE regress_superuser WITH SUPERUSER; +GRANT mdb_superuser TO regress_mdb_superuser_user1; +GRANT CREATE ON DATABASE regression TO regress_mdb_superuser_user2; +SET ROLE regress_superuser; +CREATE TABLE regress_superuser_table(); +SET ROLE pg_read_server_files; +CREATE TABLE regress_pgrsf_table(); +SET ROLE pg_write_server_files; +CREATE TABLE regress_pgwsf_table(); +SET ROLE pg_execute_server_program; +CREATE TABLE regress_pgxsp_table(); +SET ROLE pg_read_all_data; +CREATE TABLE regress_pgrad_table(); +SET ROLE pg_write_all_data; +CREATE TABLE regress_pgrwd_table(); +SET ROLE regress_mdb_superuser_user1; +CREATE TABLE regress_mdbsu_table(); +SET ROLE regress_mdb_superuser_user2; +CREATE TABLE regress_mdbsu_table2(); +SET ROLE mdb_read_all_data; +-- cannot read all data (fail) +TABLE pg_authid; +ERROR: permission denied for table pg_authid +TABLE regress_superuser_table; +ERROR: permission denied for table regress_superuser_table +TABLE regress_pgrsf_table; +ERROR: permission denied for table regress_pgrsf_table +TABLE regress_pgwsf_table; +ERROR: permission denied for table regress_pgwsf_table +TABLE regress_pgxsp_table; +ERROR: permission denied for table regress_pgxsp_table +TABLE regress_pgrad_table; +ERROR: permission denied for table regress_pgrad_table +TABLE regress_pgwsf_table; +ERROR: permission denied for table regress_pgwsf_table +-- is allow to read all other data +TABLE regress_mdbsu_table; +-- +(0 rows) + +TABLE regress_mdbsu_table2; +-- +(0 rows) + +SET ROLE mdb_write_all_data; +CREATE TABLE regress_tt_dat(); +-- cannot read all data (fail) +INSERT INTO regress_superuser_table TABLE regress_tt_dat; +ERROR: permission denied for table regress_superuser_table +INSERT INTO regress_pgrsf_table TABLE regress_tt_dat; +ERROR: permission denied for table regress_pgrsf_table +INSERT INTO regress_pgwsf_table TABLE regress_tt_dat; +ERROR: permission denied for table regress_pgwsf_table +INSERT INTO regress_pgxsp_table TABLE regress_tt_dat; +ERROR: permission denied for table regress_pgxsp_table +INSERT INTO regress_pgrad_table TABLE regress_tt_dat; +ERROR: permission denied for table regress_pgrad_table +INSERT INTO regress_pgwsf_table TABLE regress_tt_dat; +ERROR: permission denied for table regress_pgwsf_table +-- is allow to read all other data +INSERT INTO regress_mdbsu_table TABLE regress_tt_dat; +INSERT INTO regress_mdbsu_table2 TABLE regress_tt_dat; +-- end tests +RESET SESSION AUTHORIZATION; +-- +DROP TABLE regress_pgrsf_table; +DROP TABLE regress_pgwsf_table; +DROP TABLE regress_pgxsp_table; +DROP TABLE regress_pgrad_table; +DROP TABLE regress_pgrwd_table; +DROP TABLE regress_mdbsu_table; +DROP TABLE regress_mdbsu_table2; +DROP TABLE regress_superuser_table; +REVOKE CREATE ON DATABASE regression FROM regress_mdb_superuser_user2; +REVOKE CREATE ON DATABASE regression FROM regress_mdb_superuser_user3; +DROP ROLE regress_mdb_superuser_user1; +DROP ROLE regress_mdb_superuser_user2; +DROP ROLE regress_mdb_superuser_user3; +DROP ROLE regress_superuser; diff --git a/src/test/regress/expected/mdb_superuser.out b/src/test/regress/expected/mdb_superuser.out index 7849311cd1d..82de2e63461 100644 --- a/src/test/regress/expected/mdb_superuser.out +++ b/src/test/regress/expected/mdb_superuser.out @@ -4,7 +4,6 @@ CREATE ROLE regress_mdb_superuser_user3; CREATE ROLE regress_mdb_su_role_o1; CREATE ROLE regress_mdb_su_role_o2; GRANT mdb_superuser TO regress_mdb_su_role_o1; -GRANT mdb_admin TO mdb_superuser; CREATE ROLE regress_superuser WITH SUPERUSER; GRANT mdb_superuser TO regress_mdb_superuser_user1; GRANT CREATE ON DATABASE regression TO regress_mdb_superuser_user2; @@ -59,7 +58,7 @@ DROP TABLE regress_pgwsf_table; DROP TABLE regress_pgxsp_table; DROP TABLE regress_pgrad_table; DROP TABLE regress_pgrwd_table; --- does allowed to creare database, role or extension +-- does NOT allowed to create database, role or extension -- or grant such priviledge CREATE DATABASE regress_db_fail; ERROR: permission denied to create database @@ -128,3 +127,4 @@ DROP SCHEMA regress_mdb_superuser_schema; DROP ROLE regress_mdb_superuser_user1; DROP ROLE regress_mdb_superuser_user2; DROP ROLE regress_mdb_superuser_user3; +DROP ROLE regress_superuser; diff --git a/src/test/regress/expected/test_setup.out b/src/test/regress/expected/test_setup.out index da37e9d1304..a36f7a5fc3b 100644 --- a/src/test/regress/expected/test_setup.out +++ b/src/test/regress/expected/test_setup.out @@ -237,5 +237,8 @@ create function fipshash(text) return substr(encode(sha256($1::bytea), 'hex'), 1, 32); CREATE ROLE mdb_admin; CREATE ROLE mdb_superuser; +CREATE ROLE mdb_read_all_data; +CREATE ROLE mdb_write_all_data; CREATE ROLE mdb_replication; +GRANT mdb_admin TO mdb_superuser; GRANT pg_create_subscription TO mdb_admin; diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule index f0de2bca64f..d19ccd3e914 100644 --- a/src/test/regress/parallel_schedule +++ b/src/test/regress/parallel_schedule @@ -21,6 +21,8 @@ test: mdb_replication test: mdb_copy +test: mdb_read_write_roles + # ---------- # The first group of parallel tests # ---------- diff --git a/src/test/regress/sql/mdb_read_write_roles.sql b/src/test/regress/sql/mdb_read_write_roles.sql new file mode 100644 index 00000000000..291f63297a1 --- /dev/null +++ b/src/test/regress/sql/mdb_read_write_roles.sql @@ -0,0 +1,89 @@ +CREATE ROLE regress_mdb_superuser_user1; +CREATE ROLE regress_mdb_superuser_user2; +CREATE ROLE regress_mdb_superuser_user3; + +CREATE ROLE regress_superuser WITH SUPERUSER; + +GRANT mdb_superuser TO regress_mdb_superuser_user1; + +GRANT CREATE ON DATABASE regression TO regress_mdb_superuser_user2; + +SET ROLE regress_superuser; +CREATE TABLE regress_superuser_table(); + +SET ROLE pg_read_server_files; +CREATE TABLE regress_pgrsf_table(); + +SET ROLE pg_write_server_files; +CREATE TABLE regress_pgwsf_table(); + +SET ROLE pg_execute_server_program; +CREATE TABLE regress_pgxsp_table(); + +SET ROLE pg_read_all_data; +CREATE TABLE regress_pgrad_table(); + +SET ROLE pg_write_all_data; +CREATE TABLE regress_pgrwd_table(); + +SET ROLE regress_mdb_superuser_user1; +CREATE TABLE regress_mdbsu_table(); + +SET ROLE regress_mdb_superuser_user2; +CREATE TABLE regress_mdbsu_table2(); + +SET ROLE mdb_read_all_data; +-- cannot read all data (fail) +TABLE pg_authid; +TABLE regress_superuser_table; +TABLE regress_pgrsf_table; +TABLE regress_pgwsf_table; +TABLE regress_pgxsp_table; +TABLE regress_pgrad_table; +TABLE regress_pgwsf_table; + + +-- is allow to read all other data + +TABLE regress_mdbsu_table; +TABLE regress_mdbsu_table2; + + +SET ROLE mdb_write_all_data; +CREATE TABLE regress_tt_dat(); +-- cannot read all data (fail) +INSERT INTO regress_superuser_table TABLE regress_tt_dat; +INSERT INTO regress_pgrsf_table TABLE regress_tt_dat; +INSERT INTO regress_pgwsf_table TABLE regress_tt_dat; +INSERT INTO regress_pgxsp_table TABLE regress_tt_dat; +INSERT INTO regress_pgrad_table TABLE regress_tt_dat; +INSERT INTO regress_pgwsf_table TABLE regress_tt_dat; + + +-- is allow to read all other data + +INSERT INTO regress_mdbsu_table TABLE regress_tt_dat; +INSERT INTO regress_mdbsu_table2 TABLE regress_tt_dat; + +-- end tests + +RESET SESSION AUTHORIZATION; +-- + +DROP TABLE regress_pgrsf_table; +DROP TABLE regress_pgwsf_table; +DROP TABLE regress_pgxsp_table; +DROP TABLE regress_pgrad_table; +DROP TABLE regress_pgrwd_table; + +DROP TABLE regress_mdbsu_table; +DROP TABLE regress_mdbsu_table2; +DROP TABLE regress_superuser_table; + +REVOKE CREATE ON DATABASE regression FROM regress_mdb_superuser_user2; +REVOKE CREATE ON DATABASE regression FROM regress_mdb_superuser_user3; + +DROP ROLE regress_mdb_superuser_user1; +DROP ROLE regress_mdb_superuser_user2; +DROP ROLE regress_mdb_superuser_user3; +DROP ROLE regress_superuser; diff --git a/src/test/regress/sql/mdb_superuser.sql b/src/test/regress/sql/mdb_superuser.sql index 0877d9258f6..cc5caf531b3 100644 --- a/src/test/regress/sql/mdb_superuser.sql +++ b/src/test/regress/sql/mdb_superuser.sql @@ -7,8 +7,6 @@ CREATE ROLE regress_mdb_su_role_o2; GRANT mdb_superuser TO regress_mdb_su_role_o1; -GRANT mdb_admin TO mdb_superuser; - CREATE ROLE regress_superuser WITH SUPERUSER; GRANT mdb_superuser TO regress_mdb_superuser_user1; @@ -90,7 +88,7 @@ DROP TABLE regress_pgrad_table; DROP TABLE regress_pgrwd_table; --- does allowed to creare database, role or extension +-- does NOT allowed to create database, role or extension -- or grant such priviledge CREATE DATABASE regress_db_fail; @@ -162,3 +160,4 @@ DROP SCHEMA regress_mdb_superuser_schema; DROP ROLE regress_mdb_superuser_user1; DROP ROLE regress_mdb_superuser_user2; DROP ROLE regress_mdb_superuser_user3; +DROP ROLE regress_superuser; diff --git a/src/test/regress/sql/test_setup.sql b/src/test/regress/sql/test_setup.sql index 8fd2b005bc6..c78f2f5622a 100644 --- a/src/test/regress/sql/test_setup.sql +++ b/src/test/regress/sql/test_setup.sql @@ -292,6 +292,9 @@ create function fipshash(text) CREATE ROLE mdb_admin; CREATE ROLE mdb_superuser; +CREATE ROLE mdb_read_all_data; +CREATE ROLE mdb_write_all_data; CREATE ROLE mdb_replication; +GRANT mdb_admin TO mdb_superuser; GRANT pg_create_subscription TO mdb_admin; From 8629d1ca7eef5c1cd0e4169288fcc04a278e9130 Mon Sep 17 00:00:00 2001 From: reshke Date: Sat, 26 Apr 2025 12:05:23 +0500 Subject: [PATCH 32/45] Allow usage on schema for mdb_read_all_data (#23) --- src/backend/catalog/aclchk.c | 10 ++++++++++ .../regress/expected/mdb_read_write_roles.out | 15 +++++++++++++++ src/test/regress/sql/mdb_read_write_roles.sql | 12 ++++++++++++ 3 files changed, 37 insertions(+) diff --git a/src/backend/catalog/aclchk.c b/src/backend/catalog/aclchk.c index 20049ad7617..5a315b5ac8a 100644 --- a/src/backend/catalog/aclchk.c +++ b/src/backend/catalog/aclchk.c @@ -3645,6 +3645,7 @@ pg_namespace_aclmask_ext(Oid nsp_oid, Oid roleid, bool isNull; Acl *acl; Oid ownerId; + Oid mdb_read_all_data_oid; /* Superusers bypass all permission checking. */ if (superuser_arg(roleid)) @@ -3698,6 +3699,8 @@ pg_namespace_aclmask_ext(Oid nsp_oid, Oid roleid, ownerId = ((Form_pg_namespace) GETSTRUCT(tuple))->nspowner; + mdb_read_all_data_oid = get_role_oid("mdb_read_all_data", true); + aclDatum = SysCacheGetAttr(NAMESPACEOID, tuple, Anum_pg_namespace_nspacl, &isNull); if (isNull) @@ -3730,6 +3733,13 @@ pg_namespace_aclmask_ext(Oid nsp_oid, Oid roleid, (has_privs_of_role(roleid, ROLE_PG_READ_ALL_DATA) || has_privs_of_role(roleid, ROLE_PG_WRITE_ALL_DATA))) result |= ACL_USAGE; + + + if (mask & ACL_USAGE && !(result & ACL_USAGE) && + has_privs_of_role(roleid, mdb_read_all_data_oid) && + !has_privs_of_unwanted_system_role(ownerId, true)) + result |= ACL_USAGE; + return result; } diff --git a/src/test/regress/expected/mdb_read_write_roles.out b/src/test/regress/expected/mdb_read_write_roles.out index 080e95cafb9..4c8b04eec89 100644 --- a/src/test/regress/expected/mdb_read_write_roles.out +++ b/src/test/regress/expected/mdb_read_write_roles.out @@ -18,8 +18,10 @@ SET ROLE pg_write_all_data; CREATE TABLE regress_pgrwd_table(); SET ROLE regress_mdb_superuser_user1; CREATE TABLE regress_mdbsu_table(); +CREATE SCHEMA regress_schema CREATE TABLE regress_mdbsu_table(); SET ROLE regress_mdb_superuser_user2; CREATE TABLE regress_mdbsu_table2(); +CREATE SCHEMA regress_schema2 CREATE TABLE regress_mdbsu_table2(); SET ROLE mdb_read_all_data; -- cannot read all data (fail) TABLE pg_authid; @@ -45,6 +47,15 @@ TABLE regress_mdbsu_table2; -- (0 rows) +-- check USAGE of schema +TABLE regress_schema.regress_mdbsu_table; +-- +(0 rows) + +TABLE regress_schema2.regress_mdbsu_table2; +-- +(0 rows) + SET ROLE mdb_write_all_data; CREATE TABLE regress_tt_dat(); -- cannot read all data (fail) @@ -73,6 +84,10 @@ DROP TABLE regress_pgrad_table; DROP TABLE regress_pgrwd_table; DROP TABLE regress_mdbsu_table; DROP TABLE regress_mdbsu_table2; +DROP TABLE regress_schema.regress_mdbsu_table; +DROP TABLE regress_schema2.regress_mdbsu_table2; +DROP SCHEMA regress_schema; +DROP SCHEMA regress_schema2; DROP TABLE regress_superuser_table; REVOKE CREATE ON DATABASE regression FROM regress_mdb_superuser_user2; REVOKE CREATE ON DATABASE regression FROM regress_mdb_superuser_user3; diff --git a/src/test/regress/sql/mdb_read_write_roles.sql b/src/test/regress/sql/mdb_read_write_roles.sql index 291f63297a1..0522e45a631 100644 --- a/src/test/regress/sql/mdb_read_write_roles.sql +++ b/src/test/regress/sql/mdb_read_write_roles.sql @@ -28,9 +28,11 @@ CREATE TABLE regress_pgrwd_table(); SET ROLE regress_mdb_superuser_user1; CREATE TABLE regress_mdbsu_table(); +CREATE SCHEMA regress_schema CREATE TABLE regress_mdbsu_table(); SET ROLE regress_mdb_superuser_user2; CREATE TABLE regress_mdbsu_table2(); +CREATE SCHEMA regress_schema2 CREATE TABLE regress_mdbsu_table2(); SET ROLE mdb_read_all_data; -- cannot read all data (fail) @@ -48,6 +50,9 @@ TABLE regress_pgwsf_table; TABLE regress_mdbsu_table; TABLE regress_mdbsu_table2; +-- check USAGE of schema +TABLE regress_schema.regress_mdbsu_table; +TABLE regress_schema2.regress_mdbsu_table2; SET ROLE mdb_write_all_data; CREATE TABLE regress_tt_dat(); @@ -78,6 +83,13 @@ DROP TABLE regress_pgrwd_table; DROP TABLE regress_mdbsu_table; DROP TABLE regress_mdbsu_table2; + +DROP TABLE regress_schema.regress_mdbsu_table; +DROP TABLE regress_schema2.regress_mdbsu_table2; + +DROP SCHEMA regress_schema; +DROP SCHEMA regress_schema2; + DROP TABLE regress_superuser_table; REVOKE CREATE ON DATABASE regression FROM regress_mdb_superuser_user2; From 2185b97c11d68ef6dd29127940c5f7f810b3fa32 Mon Sep 17 00:00:00 2001 From: reshke Date: Sat, 26 Apr 2025 08:16:33 +0000 Subject: [PATCH 33/45] Add check for mdb_service_auth role --- src/backend/catalog/aclchk.c | 4 ++-- src/backend/utils/adt/acl.c | 20 +++++++++++++++----- src/include/utils/acl.h | 2 +- 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/src/backend/catalog/aclchk.c b/src/backend/catalog/aclchk.c index 5a315b5ac8a..1e4e4f54786 100644 --- a/src/backend/catalog/aclchk.c +++ b/src/backend/catalog/aclchk.c @@ -3392,7 +3392,7 @@ pg_class_aclmask_ext(Oid table_oid, Oid roleid, AclMode mask, */ if (mask & ACL_SELECT && !(result & ACL_SELECT) && has_privs_of_role(roleid, mdb_read_all_data_oid) && - !has_privs_of_unwanted_system_role(ownerId)) + !has_privs_of_unwanted_system_role(ownerId, true)) result |= ACL_SELECT; /* @@ -3417,7 +3417,7 @@ pg_class_aclmask_ext(Oid table_oid, Oid roleid, AclMode mask, if (mask & (ACL_INSERT | ACL_UPDATE | ACL_DELETE) && !(result & (ACL_INSERT | ACL_UPDATE | ACL_DELETE)) && has_privs_of_role(roleid, mdb_write_all_data_oid) && - !has_privs_of_unwanted_system_role(ownerId)) + !has_privs_of_unwanted_system_role(ownerId, true)) result |= (mask & (ACL_INSERT | ACL_UPDATE | ACL_DELETE)); /* diff --git a/src/backend/utils/adt/acl.c b/src/backend/utils/adt/acl.c index 4271a4152ea..dfcc5149cce 100644 --- a/src/backend/utils/adt/acl.c +++ b/src/backend/utils/adt/acl.c @@ -5337,7 +5337,9 @@ has_privs_of_role_strict_no_cache(Oid member, Oid role) */ bool -has_privs_of_unwanted_system_role(Oid role) { +has_privs_of_unwanted_system_role(Oid role, bool check_mdb_service_auth) { + Oid mdb_service_authoid; + if (has_privs_of_role_strict(role, ROLE_PG_READ_SERVER_FILES)) { return true; } @@ -5354,6 +5356,14 @@ has_privs_of_unwanted_system_role(Oid role) { return true; } + if (check_mdb_service_auth) { + mdb_service_authoid = get_role_oid("mdb_service_auth", true); + + if (has_privs_of_role_strict(role, mdb_service_authoid)) { + return true; + } + } + return false; } @@ -5400,7 +5410,7 @@ has_privs_of_role(Oid member, Oid role) * if target role is neither superuser nor * some dangerous system role */ - if (!has_privs_of_unwanted_system_role(role)) { + if (!has_privs_of_unwanted_system_role(role, true)) { return true; } } @@ -5455,7 +5465,7 @@ mdb_admin_allow_bypass_owner_checks(Oid userId, Oid ownerId) */ /* All checks passed, hope will not be hacked here (again) */ - return !has_privs_of_unwanted_system_role(ownerId); + return !has_privs_of_unwanted_system_role(ownerId, true); } // -- non-upstream patch end @@ -5534,7 +5544,7 @@ check_mdb_admin_is_member_of_role(Oid member, Oid role) GetUserNameFromId(role, false)))); } - if (has_privs_of_unwanted_system_role(role)) { + if (has_privs_of_unwanted_system_role(role, true)) { ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("forbidden to transfer ownership to this system role in Cloud"))); @@ -5568,7 +5578,7 @@ bool mdb_admin_is_member_of_role(Oid member, Oid role) { return false; } - return !has_privs_of_unwanted_system_role(role); + return !has_privs_of_unwanted_system_role(role, true); } // -- mdb admin patch diff --git a/src/include/utils/acl.h b/src/include/utils/acl.h index 913320706b4..2ddefefece2 100644 --- a/src/include/utils/acl.h +++ b/src/include/utils/acl.h @@ -231,7 +231,7 @@ extern bool mdb_admin_is_member_of_role(Oid member, Oid role); extern Oid select_best_admin(Oid member, Oid role); extern Oid get_role_oid(const char *rolname, bool missing_ok); extern Oid get_role_oid_or_public(const char *rolname); -extern bool has_privs_of_unwanted_system_role(Oid role); +extern bool has_privs_of_unwanted_system_role(Oid role, bool check_mdb_service_auth); extern Oid get_rolespec_oid(const RoleSpec *role, bool missing_ok); extern void check_rolespec_name(const RoleSpec *role, const char *detail_msg); extern HeapTuple get_rolespec_tuple(const RoleSpec *role); From 5249b9073f57ec2b666348e89b038d244dea73ef Mon Sep 17 00:00:00 2001 From: Andrey Borodin Date: Wed, 13 Nov 2024 12:03:52 +0500 Subject: [PATCH 34/45] GUCify NUM_BUFFER_PARTITIONS Also truncate query to be logged in simple query Remove unused functions in buf_internals.h --- src/backend/storage/buffer/bufmgr.c | 8 ++++++++ src/backend/utils/init/globals.c | 4 ++++ src/backend/utils/misc/guc_tables.c | 11 +++++++++++ src/backend/utils/misc/postgresql.conf.sample | 1 + src/include/storage/buf_internals.h | 14 +------------- src/include/storage/lwlock.h | 4 +++- src/include/utils/guc_hooks.h | 1 + src/test/modules/test_misc/t/003_check_guc.pl | 1 + 8 files changed, 30 insertions(+), 14 deletions(-) diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c index 0212cab1026..4d14b7e4363 100644 --- a/src/backend/storage/buffer/bufmgr.c +++ b/src/backend/storage/buffer/bufmgr.c @@ -61,6 +61,7 @@ #include "storage/read_stream.h" #include "storage/smgr.h" #include "storage/standby.h" +#include "utils/guc_hooks.h" #include "utils/memdebug.h" #include "utils/ps_status.h" #include "utils/rel.h" @@ -146,6 +147,13 @@ int bgwriter_lru_maxpages = 100; double bgwriter_lru_multiplier = 2.0; bool track_io_timing = false; +/* GUC assign hook for num_buffer_partitions_log2 */ +void +assign_num_buffer_partitions_log2(int newval, void *extra) +{ + num_buffer_partitions_mask = (1 << newval) - 1; +} + /* * How many buffers PrefetchBuffer callers should try to stay ahead of their * ReadBuffer calls by. Zero means "never prefetch". This value is only used diff --git a/src/backend/utils/init/globals.c b/src/backend/utils/init/globals.c index aaa48890aba..b513b75645d 100644 --- a/src/backend/utils/init/globals.c +++ b/src/backend/utils/init/globals.c @@ -168,3 +168,7 @@ int notify_buffers = 16; int serializable_buffers = 32; int subtransaction_buffers = 0; int transaction_buffers = 0; + +/* shared buffers partitions number and mask */ +int num_buffer_partitions_log2 = 7; +int num_buffer_partitions_mask = 127; \ No newline at end of file diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index 0356ebac4e0..96f25e57e09 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -2648,6 +2648,17 @@ struct config_int ConfigureNamesInt[] = check_max_stack_depth, assign_max_stack_depth, NULL }, + { + /* MDB prefix here is needed in case when vanilla is starting with config with this value */ + {"ycmdb.num_buffer_partitions_log2", PGC_POSTMASTER, RESOURCES_MEM, + gettext_noop("Sets number of partitions for shared buffers mapping hashtable."), + NULL, + }, + &num_buffer_partitions_log2, + 7, 5, 16, + NULL, assign_num_buffer_partitions_log2, NULL + }, + { {"temp_file_limit", PGC_SUSET, RESOURCES_DISK, gettext_noop("Limits the total size of all temporary files used by each process."), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index 8df2b6fbb3b..207b4b8e31c 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -146,6 +146,7 @@ #autovacuum_work_mem = -1 # min 64kB, or -1 to use maintenance_work_mem #logical_decoding_work_mem = 64MB # min 64kB #max_stack_depth = 2MB # min 100kB +#mdb.num_buffer_partitions_log2 = 7 # number of partitions in shared buffer mapping hashtable #shared_memory_type = mmap # the default is the first option # supported by the operating system: # mmap diff --git a/src/include/storage/buf_internals.h b/src/include/storage/buf_internals.h index 0dec7d93b3b..04d4cdffbd0 100644 --- a/src/include/storage/buf_internals.h +++ b/src/include/storage/buf_internals.h @@ -189,23 +189,11 @@ BufTagMatchesRelFileLocator(const BufferTag *tag, * hash code with BufTableHashCode(), then apply BufMappingPartitionLock(). * NB: NUM_BUFFER_PARTITIONS must be a power of 2! */ -static inline uint32 -BufTableHashPartition(uint32 hashcode) -{ - return hashcode % NUM_BUFFER_PARTITIONS; -} - static inline LWLock * BufMappingPartitionLock(uint32 hashcode) { return &MainLWLockArray[BUFFER_MAPPING_LWLOCK_OFFSET + - BufTableHashPartition(hashcode)].lock; -} - -static inline LWLock * -BufMappingPartitionLockByIndex(uint32 index) -{ - return &MainLWLockArray[BUFFER_MAPPING_LWLOCK_OFFSET + index].lock; + (hashcode & num_buffer_partitions_mask)].lock; } /* diff --git a/src/include/storage/lwlock.h b/src/include/storage/lwlock.h index 08a72569ae5..03502fca4ae 100644 --- a/src/include/storage/lwlock.h +++ b/src/include/storage/lwlock.h @@ -90,7 +90,9 @@ extern PGDLLIMPORT int NamedLWLockTrancheRequests; */ /* Number of partitions of the shared buffer mapping hashtable */ -#define NUM_BUFFER_PARTITIONS 128 +extern int num_buffer_partitions_log2; +extern int num_buffer_partitions_mask; +#define NUM_BUFFER_PARTITIONS (1 << num_buffer_partitions_log2) /* Number of partitions the shared lock tables are divided into */ #define LOG2_NUM_LOCK_PARTITIONS 4 diff --git a/src/include/utils/guc_hooks.h b/src/include/utils/guc_hooks.h index 82ac8646a8d..974ddb46360 100644 --- a/src/include/utils/guc_hooks.h +++ b/src/include/utils/guc_hooks.h @@ -87,6 +87,7 @@ extern void assign_io_combine_limit(int newval, void *extra); extern void assign_max_wal_size(int newval, void *extra); extern bool check_max_stack_depth(int *newval, void **extra, GucSource source); extern void assign_max_stack_depth(int newval, void *extra); +extern void assign_num_buffer_partitions_log2(int newval, void *extra); extern bool check_multixact_member_buffers(int *newval, void **extra, GucSource source); extern bool check_multixact_offset_buffers(int *newval, void **extra, diff --git a/src/test/modules/test_misc/t/003_check_guc.pl b/src/test/modules/test_misc/t/003_check_guc.pl index 1df531b03ef..364ed02e74b 100644 --- a/src/test/modules/test_misc/t/003_check_guc.pl +++ b/src/test/modules/test_misc/t/003_check_guc.pl @@ -73,6 +73,7 @@ } push @gucs_in_file, "ycmdb.yc_grant_checker"; +push @gucs_in_file, "ycmdb.num_buffer_partitions_log2"; close $contents; From 0b1136a99339c5c9d77712bfb4e95c4c31fed882 Mon Sep 17 00:00:00 2001 From: reshke Date: Wed, 6 Aug 2025 12:55:27 +0000 Subject: [PATCH 35/45] deb fix --- debian/libpq5.symbols | 1 - 1 file changed, 1 deletion(-) diff --git a/debian/libpq5.symbols b/debian/libpq5.symbols index 488be5f18cb..b5df7668a60 100644 --- a/debian/libpq5.symbols +++ b/debian/libpq5.symbols @@ -147,7 +147,6 @@ libpq.so.5 libpq5 #MINVER# PQsendQueryParams@Base 0 PQsendQueryPrepared@Base 0 PQserverVersion@Base 0 - PQservice@Base 18~~ PQsetAuthDataHook@Base 18~~ PQsetChunkedRowsMode@Base 17~~ PQsetClientEncoding@Base 0 From 2cab50b6065af9c6969931fa208655ce669f745e Mon Sep 17 00:00:00 2001 From: Roman Khapov Date: Thu, 2 Oct 2025 11:14:46 +0500 Subject: [PATCH 36/45] pg_stat_statements.c: cancelable qtext_load_file (#45) In the case of a large PGSS_TEXT_FILE, the work time of the qtext_load_file function will be quite long, and the query to the pg_stat_statements table will not be cancellable, as there is no CHECK_FOR_INTERRUPT in the function. Also, the amount of bytes read can reach 1 GB, which leads to a slow read system call that does not allow cancellation of the query. Testing the speed of sequential read using fio with different block sizes shows that there is no significant difference between 16 MB blocks and 1 GB blocks. Therefore, this patch changes the maximum read value from 1 GB to 16 MB and adds INTERRUPTS_PENDING_CONDITION() check in the read loop of qtext_load_file to make it cancellable. For now, only statement execution is cancellable (fail_on_interrupt is true only for calls from pg_stat_statements_internal) Signed-off-by: rkhapov Reviewed-by: reshke --- .../pg_stat_statements/pg_stat_statements.c | 29 ++++++++++++++----- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/contrib/pg_stat_statements/pg_stat_statements.c b/contrib/pg_stat_statements/pg_stat_statements.c index 77450947bee..14d2d158795 100644 --- a/contrib/pg_stat_statements/pg_stat_statements.c +++ b/contrib/pg_stat_statements/pg_stat_statements.c @@ -365,7 +365,7 @@ static pgssEntry *entry_alloc(pgssHashKey *key, Size query_offset, int query_len static void entry_dealloc(void); static bool qtext_store(const char *query, int query_len, Size *query_offset, int *gc_count); -static char *qtext_load_file(Size *buffer_size); +static char *qtext_load_file(Size *buffer_size, bool fail_on_interrupts); static char *qtext_fetch(Size query_offset, int query_len, char *buffer, Size buffer_size); static bool need_gc_qtexts(void); @@ -767,7 +767,7 @@ pgss_shmem_shutdown(int code, Datum arg) if (fwrite(&num_entries, sizeof(int32), 1, file) != 1) goto error; - qbuffer = qtext_load_file(&qbuffer_size); + qbuffer = qtext_load_file(&qbuffer_size, false /* fail_on_interrupts */ ); if (qbuffer == NULL) goto error; @@ -1772,7 +1772,7 @@ pg_stat_statements_internal(FunctionCallInfo fcinfo, /* No point in loading file now if there are active writers */ if (n_writers == 0) - qbuffer = qtext_load_file(&qbuffer_size); + qbuffer = qtext_load_file(&qbuffer_size, true /* fail_on_interrupts */ ); } /* @@ -1806,7 +1806,7 @@ pg_stat_statements_internal(FunctionCallInfo fcinfo, { if (qbuffer) pfree(qbuffer); - qbuffer = qtext_load_file(&qbuffer_size); + qbuffer = qtext_load_file(&qbuffer_size, true /* fail_on_interrupts */ ); } } @@ -1825,6 +1825,12 @@ pg_stat_statements_internal(FunctionCallInfo fcinfo, memset(values, 0, sizeof(values)); memset(nulls, 0, sizeof(nulls)); + /* Can't process interrupts here - pgss-lock is acquired */ + if (INTERRUPTS_PENDING_CONDITION()) + { + break; + } + values[i++] = ObjectIdGetDatum(entry->key.userid); values[i++] = ObjectIdGetDatum(entry->key.dbid); if (api_version >= PGSS_V1_9) @@ -2020,6 +2026,8 @@ pg_stat_statements_internal(FunctionCallInfo fcinfo, LWLockRelease(pgss->lock); + CHECK_FOR_INTERRUPTS(); + if (qbuffer) pfree(qbuffer); } @@ -2319,7 +2327,7 @@ qtext_store(const char *query, int query_len, * the caller is responsible for verifying that the result is sane. */ static char * -qtext_load_file(Size *buffer_size) +qtext_load_file(Size *buffer_size, bool fail_on_interrupts) { char *buf; int fd; @@ -2372,7 +2380,14 @@ qtext_load_file(Size *buffer_size) nread = 0; while (nread < stat.st_size) { - int toread = Min(1024 * 1024 * 1024, stat.st_size - nread); + int toread = Min(32 * 1024 * 1024, stat.st_size - nread); + + if (fail_on_interrupts && INTERRUPTS_PENDING_CONDITION()) + { + free(buf); + CloseTransientFile(fd); + return NULL; + } /* * If we get a short read and errno doesn't get set, the reason is @@ -2509,7 +2524,7 @@ gc_qtexts(void) * file is only going to get bigger; hoping for a future non-OOM result is * risky and can easily lead to complete denial of service. */ - qbuffer = qtext_load_file(&qbuffer_size); + qbuffer = qtext_load_file(&qbuffer_size, false /* fail_on_interrupts */ ); if (qbuffer == NULL) goto gc_fail; From b8f1db854ab8c6a9e0c01a0113cc4336fb9d4f36 Mon Sep 17 00:00:00 2001 From: reshke Date: Fri, 5 Dec 2025 08:46:56 +0000 Subject: [PATCH 37/45] Allow canceling service queries. Allow user to cancel backend which is explicitly marked as service operation, allowed to be canceled. We check `application_name` for containing MDB prefix Fix MDB-signal process and add TAP test for feature --- src/backend/storage/ipc/signalfuncs.c | 27 +++++++++- src/test/mdb_admin/t/mdb_app_name.pl | 75 +++++++++++++++++++++++++++ 2 files changed, 101 insertions(+), 1 deletion(-) create mode 100644 src/test/mdb_admin/t/mdb_app_name.pl diff --git a/src/backend/storage/ipc/signalfuncs.c b/src/backend/storage/ipc/signalfuncs.c index 3c74ba9b3cb..8596816e6eb 100644 --- a/src/backend/storage/ipc/signalfuncs.c +++ b/src/backend/storage/ipc/signalfuncs.c @@ -53,8 +53,10 @@ pg_signal_backend(int pid, int sig) { PGPROC *proc = BackendPidGetProc(pid); LocalPgBackendStatus *local_beentry; + bool ok; local_beentry = NULL; + ok = false; /* * BackendPidGetProc returns NULL if the pid isn't valid; but by the time @@ -89,7 +91,30 @@ pg_signal_backend(int pid, int sig) * * Otherwise, users can signal backends for roles they have privileges of. */ - if (!OidIsValid(proc->roleId) || superuser_arg(proc->roleId)) + + /* We allow to kill queries issued with MDB prefix */ + if (!superuser() && OidIsValid(proc->roleId)) + { + /* Ok, we are not superuser, and we try to kill + * some backend, which can may be a MDB-cancellable query*/ + LocalPgBackendStatus *local_beentry; + char * appname = NULL; + local_beentry = pgstat_get_local_beentry_by_proc_number(GetNumberFromPGProc(proc)); + + if (local_beentry != NULL) + { + appname = local_beentry->backendStatus.st_appname; + if (appname != NULL && strncmp(appname, "MDB", 3) == 0) + ok = true; + } + } + + if (ok) + { + /* This code style is not very conventional, but for sake of rebase, we code it like this */ + ; + } + else if (!OidIsValid(proc->roleId) || superuser_arg(proc->roleId)) { ProcNumber procNumber = GetNumberFromPGProc(proc); BackendType backendType = pgstat_get_backend_type_by_proc_number(procNumber); diff --git a/src/test/mdb_admin/t/mdb_app_name.pl b/src/test/mdb_admin/t/mdb_app_name.pl new file mode 100644 index 00000000000..9335337fb42 --- /dev/null +++ b/src/test/mdb_admin/t/mdb_app_name.pl @@ -0,0 +1,75 @@ + +# Copyright (c) 2024-2025, MDB, Mother Russia + +# Minimal test testing streaming replication +use strict; +use warnings; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +# Initialize primary node +my $node_primary = PostgreSQL::Test::Cluster->new('primary'); +$node_primary->init(); +$node_primary->start; + +# Create some content on primary and check its presence in standby nodes +$node_primary->safe_psql('postgres', + " + CREATE DATABASE regress; + CREATE ROLE mdb_reg_lh_app_name; + GRANT pg_signal_backend to mdb_reg_lh_app_name; + CREATE TABLE mdb_app_name_t(i int); +"); + +my $main_sess = $node_primary->background_psql('postgres'); + +$main_sess->query_safe( + q( +SET application_name TO 'SU'; +BEGIN; +INSERT INTO mdb_app_name_t VALUES(0); +)); + + +my $res_pid = $node_primary->safe_psql('regress', + " + SELECT pid FROM pg_stat_activity WHERE application_name = 'SU'; +"); + +print "pid is $res_pid\n"; + +ok(1); + + +my ($res_reg_lh_1, $stdout_reg_lh_1, $stderr_reg_lh_1) = $node_primary->psql('regress', + " + SET ROLE mdb_reg_lh_app_name; + SELECT pg_terminate_backend($res_pid); +"); + +# print ($res_reg_lh_1, $stdout_reg_lh_1, $stderr_reg_lh_1, "\n"); + +ok($res_reg_lh_1 != 0, "should fail for non-MDB"); +like($stderr_reg_lh_1, qr/Only roles with the SUPERUSER attribute may terminate processes of roles with the SUPERUSER attribute./, "matches"); + +# should succeed +$main_sess->query_safe(qq[COMMIT]); + +$main_sess->query_safe( + q( +SET application_name TO 'MDB'; +BEGIN; +INSERT INTO mdb_app_name_t VALUES(1); +)); + + +my ($res_reg_lh_2, $stdout_reg_lh_2, $stderr_reg_lh_2) = $node_primary->psql('regress', + " + SET ROLE mdb_reg_lh_app_name; + SELECT pg_terminate_backend($res_pid); +"); + +ok($res_reg_lh_2 == 0, "should success for MDB"); + +done_testing(); From f86c18dc811d78181fe5301610405546ed107531 Mon Sep 17 00:00:00 2001 From: reshke Date: Sun, 18 Jan 2026 14:12:21 +0000 Subject: [PATCH 38/45] v6 of bt_page_items pretty-print --- contrib/pageinspect/Makefile | 2 +- contrib/pageinspect/btreefuncs.c | 142 ++++++++++++++++-- contrib/pageinspect/expected/btree.out | 25 +++ .../pageinspect/pageinspect--1.13--1.14.sql | 20 +++ contrib/pageinspect/pageinspect.control | 2 +- contrib/pageinspect/sql/btree.sql | 4 + 6 files changed, 183 insertions(+), 12 deletions(-) create mode 100644 contrib/pageinspect/pageinspect--1.13--1.14.sql diff --git a/contrib/pageinspect/Makefile b/contrib/pageinspect/Makefile index 9dee7653310..b5565c46cf1 100644 --- a/contrib/pageinspect/Makefile +++ b/contrib/pageinspect/Makefile @@ -13,7 +13,7 @@ OBJS = \ rawpage.o EXTENSION = pageinspect -DATA = pageinspect--1.12--1.13.sql \ +DATA = pageinspect--1.13--1.14.sql pageinspect--1.12--1.13.sql \ pageinspect--1.11--1.12.sql pageinspect--1.10--1.11.sql \ pageinspect--1.9--1.10.sql pageinspect--1.8--1.9.sql \ pageinspect--1.7--1.8.sql pageinspect--1.6--1.7.sql \ diff --git a/contrib/pageinspect/btreefuncs.c b/contrib/pageinspect/btreefuncs.c index 294821231fc..75e97af3a38 100644 --- a/contrib/pageinspect/btreefuncs.c +++ b/contrib/pageinspect/btreefuncs.c @@ -29,6 +29,7 @@ #include "access/nbtree.h" #include "access/relation.h" +#include "access/tupdesc.h" #include "catalog/namespace.h" #include "catalog/pg_am.h" #include "catalog/pg_type.h" @@ -38,6 +39,8 @@ #include "utils/array.h" #include "utils/builtins.h" #include "utils/rel.h" +#include "utils/ruleutils.h" +#include "utils/lsyscache.h" #include "utils/varlena.h" PG_FUNCTION_INFO_V1(bt_metap); @@ -95,6 +98,8 @@ typedef struct ua_page_items bool leafpage; bool rightmost; TupleDesc tupd; + Relation indexRel; + bool pretty_print; } ua_page_items; @@ -546,17 +551,118 @@ bt_page_print_tuples(ua_page_items *uargs) if (dlen < 0 || dlen > INDEX_SIZE_MASK) elog(ERROR, "invalid tuple length %d for tuple at offset number %u", dlen, offset); - dump = palloc0(dlen * 3 + 1); - datacstring = dump; - for (off = 0; off < dlen; off++) + + if (!uargs->pretty_print) { - if (off > 0) - *dump++ = ' '; - sprintf(dump, "%02x", *(ptr + off) & 0xff); - dump += 2; + /* Old-style, print hex bytes */ + dump = palloc0(dlen * 3 + 1); + datacstring = dump; + for (off = 0; off < dlen; off++) + { + if (off > 0) + *dump++ = ' '; + sprintf(dump, "%02x", *(ptr + off) & 0xff); + dump += 2; + } + values[j++] = CStringGetTextDatum(datacstring); + pfree(datacstring); + } + else + { + /* Do pretty-print, akin to record_out() */ + StringInfoData buf; + TupleDesc tupdesc; + + Datum itup_values[INDEX_MAX_KEYS]; + bool itup_isnull[INDEX_MAX_KEYS]; + char *index_columns; + + /* + * Included attributes are added when dealing with leaf pages, discarded + * for non-leaf pages as these include only data for key attributes. + */ + int printflags = RULE_INDEXDEF_PRETTY; + if (P_ISLEAF((BTPageOpaque) PageGetSpecialPointer(page))) + { + tupdesc = RelationGetDescr(uargs->indexRel); + } + else + { + tupdesc = CreateTupleDescCopy(RelationGetDescr(uargs->indexRel)); + tupdesc->natts = IndexRelationGetNumberOfKeyAttributes(uargs->indexRel); + printflags |= RULE_INDEXDEF_KEYS_ONLY; + } + + index_columns = pg_get_indexdef_columns_extended(RelationGetRelid(uargs->indexRel), + printflags); + + + index_deform_tuple(itup, tupdesc, + itup_values, itup_isnull); + + + initStringInfo(&buf); + appendStringInfo(&buf, "(%s)=(", index_columns); + + for (int i = 0; i < tupdesc->natts; i++) + { + char *value; + char *tmp; + bool nq = false; + + if (itup_isnull[i]) + value = "null"; + else + { + Oid foutoid; + bool typisvarlena; + Oid typoid; + + typoid = TupleDescAttr(tupdesc, i)->atttypid; + getTypeOutputInfo(typoid, &foutoid, &typisvarlena); + value = OidOutputFunctionCall(foutoid, itup_values[i]); + } + + if (i == IndexRelationGetNumberOfKeyAttributes(uargs->indexRel)) + appendStringInfoString(&buf, ") INCLUDE ("); + else if (i > 0) + appendStringInfoString(&buf, ", "); + + /* Check whether we need double quotes for this value */ + nq = (value[0] == '\0'); /* force quotes for empty string */ + for (tmp = value; *tmp; tmp++) + { + char ch = *tmp; + + if (ch == '"' || ch == '\\' || + ch == '(' || ch == ')' || ch == ',' || + isspace((unsigned char) ch)) + { + nq = true; + break; + } + } + + /* And emit the string */ + if (nq) + appendStringInfoCharMacro(&buf, '"'); + for (tmp = value; *tmp; tmp++) + { + char ch = *tmp; + + if (ch == '"' || ch == '\\') + appendStringInfoCharMacro(&buf, ch); + appendStringInfoCharMacro(&buf, ch); + } + if (nq) + appendStringInfoCharMacro(&buf, '"'); + } + + appendStringInfoChar(&buf, ')'); + + values[j++] = CStringGetTextDatum(buf.data); + pfree(buf.data); } - values[j++] = CStringGetTextDatum(datacstring); - pfree(datacstring); /* * We need to work around the BTreeTupleIsPivot() !heapkeyspace limitation @@ -628,6 +734,11 @@ bt_page_items_internal(PG_FUNCTION_ARGS, enum pageinspect_version ext_version) FuncCallContext *fctx; MemoryContext mctx; ua_page_items *uargs; + bool pretty_print = false; + + if (PG_NARGS() >= 3) { + pretty_print = PG_GETARG_BOOL(2); + } if (!superuser()) ereport(ERROR, @@ -665,7 +776,6 @@ bt_page_items_internal(PG_FUNCTION_ARGS, enum pageinspect_version ext_version) memcpy(uargs->page, BufferGetPage(buffer), BLCKSZ); UnlockReleaseBuffer(buffer); - relation_close(rel, AccessShareLock); uargs->offset = FirstOffsetNumber; @@ -681,6 +791,8 @@ bt_page_items_internal(PG_FUNCTION_ARGS, enum pageinspect_version ext_version) } uargs->leafpage = P_ISLEAF(opaque); uargs->rightmost = P_RIGHTMOST(opaque); + uargs->pretty_print = pretty_print; + uargs->indexRel = rel; /* Build a tuple descriptor for our result type */ if (get_call_result_type(fcinfo, NULL, &tupleDesc) != TYPEFUNC_COMPOSITE) @@ -704,6 +816,9 @@ bt_page_items_internal(PG_FUNCTION_ARGS, enum pageinspect_version ext_version) SRF_RETURN_NEXT(fctx, result); } + if (uargs->indexRel) + relation_close(uargs->indexRel, AccessShareLock); + SRF_RETURN_DONE(fctx); } @@ -798,6 +913,10 @@ bt_page_items_bytea(PG_FUNCTION_ARGS) uargs->leafpage = P_ISLEAF(opaque); uargs->rightmost = P_RIGHTMOST(opaque); + uargs->pretty_print = false; + /* For bytea function, we cannot do pretty-print */ + uargs->indexRel = NULL; + /* Build a tuple descriptor for our result type */ if (get_call_result_type(fcinfo, NULL, &tupleDesc) != TYPEFUNC_COMPOSITE) elog(ERROR, "return type must be a row type"); @@ -820,6 +939,9 @@ bt_page_items_bytea(PG_FUNCTION_ARGS) SRF_RETURN_NEXT(fctx, result); } + if (uargs->indexRel) + relation_close(uargs->indexRel, AccessShareLock); + SRF_RETURN_DONE(fctx); } diff --git a/contrib/pageinspect/expected/btree.out b/contrib/pageinspect/expected/btree.out index 0aa5d73322f..f407a17121b 100644 --- a/contrib/pageinspect/expected/btree.out +++ b/contrib/pageinspect/expected/btree.out @@ -1,6 +1,7 @@ CREATE TABLE test1 (a int8, b int4range); INSERT INTO test1 VALUES (72057594037927937, '[0,1)'); CREATE INDEX test1_a_idx ON test1 USING btree (a); +CREATE INDEX test1_a_b_idx ON test1 USING btree (a, b); \x SELECT * FROM bt_metap('test1_a_idx'); -[ RECORD 1 ]-------------+------- @@ -147,6 +148,30 @@ btpo_flags | 1 SELECT * FROM bt_multi_page_stats('test2_col1_idx', 7, 2); ERROR: block number 7 is out of range DROP TABLE test2; +SELECT * FROM bt_page_items('test1_a_idx', 1, true); +-[ RECORD 1 ]----------------------- +itemoffset | 1 +ctid | (0,1) +itemlen | 16 +nulls | f +vars | f +data | (a)=(72057594037927937) +dead | f +htid | (0,1) +tids | + +SELECT * FROM bt_page_items('test1_a_b_idx', 1, true); +-[ RECORD 1 ]----------------------------------- +itemoffset | 1 +ctid | (0,1) +itemlen | 32 +nulls | f +vars | t +data | (a, b)=(72057594037927937, "[0,1)") +dead | f +htid | (0,1) +tids | + SELECT * FROM bt_page_items('test1_a_idx', -1); ERROR: invalid block number -1 SELECT * FROM bt_page_items('test1_a_idx', 0); diff --git a/contrib/pageinspect/pageinspect--1.13--1.14.sql b/contrib/pageinspect/pageinspect--1.13--1.14.sql new file mode 100644 index 00000000000..874b95ed1cc --- /dev/null +++ b/contrib/pageinspect/pageinspect--1.13--1.14.sql @@ -0,0 +1,20 @@ +/* contrib/pageinspect/pageinspect--1.13--1.14.sql */ + +-- complain if script is sourced in psql, rather than via ALTER EXTENSION +\echo Use "ALTER EXTENSION pageinspect UPDATE TO '1.14'" to load this file. \quit + +DROP FUNCTION bt_page_items(text, int8); +CREATE FUNCTION bt_page_items(IN relname text, IN blkno int8, + IN pretty_print boolean DEFAULT FALSE, + OUT itemoffset smallint, + OUT ctid tid, + OUT itemlen smallint, + OUT nulls bool, + OUT vars bool, + OUT data text, + OUT dead boolean, + OUT htid tid, + OUT tids tid[]) +RETURNS SETOF record +AS 'MODULE_PATHNAME', 'bt_page_items_1_9' +LANGUAGE C STRICT PARALLEL SAFE; diff --git a/contrib/pageinspect/pageinspect.control b/contrib/pageinspect/pageinspect.control index cfc87feac03..aee3f598a9e 100644 --- a/contrib/pageinspect/pageinspect.control +++ b/contrib/pageinspect/pageinspect.control @@ -1,5 +1,5 @@ # pageinspect extension comment = 'inspect the contents of database pages at a low level' -default_version = '1.13' +default_version = '1.14' module_pathname = '$libdir/pageinspect' relocatable = true diff --git a/contrib/pageinspect/sql/btree.sql b/contrib/pageinspect/sql/btree.sql index 102ebdefe3c..36f100656a4 100644 --- a/contrib/pageinspect/sql/btree.sql +++ b/contrib/pageinspect/sql/btree.sql @@ -1,6 +1,7 @@ CREATE TABLE test1 (a int8, b int4range); INSERT INTO test1 VALUES (72057594037927937, '[0,1)'); CREATE INDEX test1_a_idx ON test1 USING btree (a); +CREATE INDEX test1_a_b_idx ON test1 USING btree (a, b); \x @@ -22,6 +23,9 @@ SELECT * FROM bt_multi_page_stats('test2_col1_idx', 3, 2); SELECT * FROM bt_multi_page_stats('test2_col1_idx', 7, 2); DROP TABLE test2; +SELECT * FROM bt_page_items('test1_a_idx', 1, true); +SELECT * FROM bt_page_items('test1_a_b_idx', 1, true); + SELECT * FROM bt_page_items('test1_a_idx', -1); SELECT * FROM bt_page_items('test1_a_idx', 0); SELECT * FROM bt_page_items('test1_a_idx', 1); From a189c4de3931c124ef4f082ee6480c54fe2bd965 Mon Sep 17 00:00:00 2001 From: Andrey Borodin Date: Tue, 10 Feb 2026 12:47:32 +0500 Subject: [PATCH 39/45] Add archive_mode=shared for coordinated WAL archiving Introduce a new archive_mode setting "shared" to prevent WAL history loss during standby promotion in HA streaming replication setups. In shared mode, the primary proactively sends archival status updates to standbys via the replication protocol. The standby creates .ready files for received WAL segments but defers marking them as .done until the primary confirms archival. This prevents WAL from being recycled before it's safely archived, addressing a critical gap in PITR continuity during failover. Key implementation details: - Primary periodically sends last archived WAL segment via new PqReplMsg_ArchiveStatusReport ('a') message - Standby marks all segments <= reported segment as .done using alphanumeric comparison on segment part (timeline-safe) - Archiver skips during recovery in shared mode, activates on promotion - Cascading replication: each standby coordinates with immediate upstream - Startup check rejects archive_mode=on during recovery This "push" design (primary sends status) is more efficient than "pull" (standby queries per-segment), avoiding directory scans and stat() calls. Based on Heikki Linnakangas's 2014 design and Greenplum's production implementation, modernized for PostgreSQL 19. Includes TAP tests covering basic synchronization, promotion, cascading replication, and multiple standbys scenarios. --- doc/src/sgml/config.sgml | 36 ++- doc/src/sgml/high-availability.sgml | 72 ++++-- src/backend/access/transam/xlog.c | 1 + src/backend/postmaster/pgarch.c | 17 +- src/backend/replication/walreceiver.c | 146 +++++++++++- src/backend/replication/walsender.c | 93 ++++++++ src/include/access/xlog.h | 1 + src/include/libpq/protocol.h | 22 ++ src/test/recovery/t/050_archive_shared.pl | 270 ++++++++++++++++++++++ 9 files changed, 620 insertions(+), 38 deletions(-) create mode 100644 src/test/recovery/t/050_archive_shared.pl diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index b7aae61af56..7051d946f8b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3832,14 +3832,36 @@ include_dir 'conf.d' are sent to archive storage by setting or . In addition to off, - to disable, there are two modes: on, and - always. During normal operation, there is no - difference between the two modes, but when set to always - the WAL archiver is enabled also during archive recovery or standby - mode. In always mode, all files restored from the archive - or streamed with streaming replication will be archived (again). See - for details. + to disable, there are three modes: on, shared, + and always. During normal operation as a primary, there is no + difference between the three modes, but they differ during archive recovery or + standby mode: + + + + on: Archives WAL only when running as a primary. + + + + + shared: Coordinates archiving between primary and standby. + The standby defers WAL archival and deletion until the primary confirms + archival via streaming replication. This prevents WAL history loss during + standby promotion in high availability setups. Upon promotion, the standby + automatically starts archiving any remaining unarchived WAL. This mode works + with cascading replication, where each standby coordinates with its immediate + upstream server. See for details. + + + + + always: Archives all WAL independently, even during recovery. + All files restored from the archive or streamed with streaming physical + replication will be archived (again), regardless of their source. + + + archive_mode is a separate setting from archive_command and diff --git a/doc/src/sgml/high-availability.sgml b/doc/src/sgml/high-availability.sgml index aa256e2ec51..35fceebfe1c 100644 --- a/doc/src/sgml/high-availability.sgml +++ b/doc/src/sgml/high-availability.sgml @@ -1385,35 +1385,61 @@ synchronous_standby_names = 'ANY 2 (s1, s2, s3)' - When continuous WAL archiving is used in a standby, there are two - different scenarios: the WAL archive can be shared between the primary - and the standby, or the standby can have its own WAL archive. When - the standby has its own WAL archive, set archive_mode + When continuous WAL archiving is used in a standby, there are three + different scenarios: the standby can have its own independent WAL archive, + the WAL archive can be shared between the primary and standby, or archiving + can be coordinated between them. + + + + For an independent archive, set archive_mode to always, and the standby will call the archive command for every WAL segment it receives, whether it's by restoring - from the archive or by streaming replication. The shared archive can - be handled similarly, but the archive_command or archive_library must - test if the file being archived exists already, and if the existing file - has identical contents. This requires more care in the - archive_command or archive_library, as it must - be careful to not overwrite an existing file with different contents, - but return success if the exactly same file is archived twice. And - all that must be done free of race conditions, if two servers attempt - to archive the same file at the same time. + from the archive or by streaming replication. + + + + For a shared archive where both primary and standby can write, use + always mode as well, but the archive_command + or archive_library must test if the file being archived + exists already, and if the existing file has identical contents. This requires + more care in the archive_command or archive_library, + as it must be careful to not overwrite an existing file with different contents, + but return success if the exactly same file is archived twice. And all that must + be done free of race conditions, if two servers attempt to archive the same file + at the same time. + + + + For coordinated archiving in high availability setups, use + archive_mode=shared. In this mode, only + the primary archives WAL segments. The standby creates .ready + files for received segments but defers actual archiving. The primary periodically + sends archival status updates to the standby via streaming replication, informing + it which segments have been archived. The standby then marks these as archived + and allows them to be recycled. Upon promotion, the standby automatically starts + archiving any remaining WAL segments that weren't confirmed as archived by the + former primary. This prevents WAL history loss during failover while avoiding + the complexity of coordinating concurrent archiving. This mode works with cascading + replication, where each standby coordinates with its immediate upstream server. If archive_mode is set to on, the - archiver is not enabled during recovery or standby mode. If the standby - server is promoted, it will start archiving after the promotion, but - will not archive any WAL or timeline history files that - it did not generate itself. To get a complete - series of WAL files in the archive, you must ensure that all WAL is - archived, before it reaches the standby. This is inherently true with - file-based log shipping, as the standby can only restore files that - are found in the archive, but not if streaming replication is enabled. - When a server is not in recovery mode, there is no difference between - on and always modes. + archiver is not enabled during recovery or standby mode, and this setting + cannot be used on a standby. If a standby with archive_mode + set to on is promoted, it will start archiving after the + promotion, but will not archive any WAL or timeline history files that it did + not generate itself. To get a complete series of WAL files in the archive, you + must ensure that all WAL is archived before it reaches the standby. This is + inherently true with file-based log shipping, as the standby can only restore + files that are found in the archive, but not if streaming replication is enabled. + + + + When a server is not in recovery mode, on, + shared, and always modes all behave + identically, archiving completed WAL segments. diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 47c04f26d40..8e6b4e0470b 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -193,6 +193,7 @@ const struct config_enum_entry archive_mode_options[] = { {"always", ARCHIVE_MODE_ALWAYS, false}, {"on", ARCHIVE_MODE_ON, false}, {"off", ARCHIVE_MODE_OFF, false}, + {"shared", ARCHIVE_MODE_SHARED, false}, {"true", ARCHIVE_MODE_ON, true}, {"false", ARCHIVE_MODE_OFF, true}, {"yes", ARCHIVE_MODE_ON, true}, diff --git a/src/backend/postmaster/pgarch.c b/src/backend/postmaster/pgarch.c index 7e622ae4bd2..bb22da47580 100644 --- a/src/backend/postmaster/pgarch.c +++ b/src/backend/postmaster/pgarch.c @@ -382,6 +382,15 @@ pgarch_ArchiverCopyLoop(void) { char xlog[MAX_XFN_CHARS + 1]; + /* + * In shared archive mode during recovery, the archiver doesn't archive + * files. The primary is responsible for archiving, and the walreceiver + * marks files as .done when the primary confirms archival. After + * promotion, the archiver starts working normally. + */ + if (XLogArchiveMode == ARCHIVE_MODE_SHARED && RecoveryInProgress()) + return; + /* force directory scan in the first call to pgarch_readyXlog() */ arch_files->arch_files_size = 0; @@ -472,10 +481,10 @@ pgarch_ArchiverCopyLoop(void) continue; } - if (pgarch_archiveXlog(xlog)) - { - /* successful */ - pgarch_archiveDone(xlog); + if (pgarch_archiveXlog(xlog)) + { + /* successful */ + pgarch_archiveDone(xlog); /* * Tell the cumulative stats system about the WAL file that we diff --git a/src/backend/replication/walreceiver.c b/src/backend/replication/walreceiver.c index 8c4d0fd9aed..524b40cdeb4 100644 --- a/src/backend/replication/walreceiver.c +++ b/src/backend/replication/walreceiver.c @@ -131,6 +131,11 @@ static TimestampTz wakeup[NUM_WALRCV_WAKEUPS]; static StringInfoData reply_message; +/* Last archived WAL segment file reported by the primary */ +static char primary_last_archived[MAX_XFN_CHARS + 1]; +static TimeLineID primary_last_archived_tli = 0; +static XLogSegNo primary_last_archived_segno = 0; + /* Prototypes for private functions */ static void WalRcvFetchTimeLineHistoryFiles(TimeLineID first, TimeLineID last); static void WalRcvWaitForStartPosition(XLogRecPtr *startpoint, TimeLineID *startpointTLI); @@ -144,6 +149,7 @@ static void XLogWalRcvClose(XLogRecPtr recptr, TimeLineID tli); static void XLogWalRcvSendReply(bool force, bool requestReply); static void XLogWalRcvSendHSFeedback(bool immed); static void ProcessWalSndrMessage(XLogRecPtr walEnd, TimestampTz sendTime); +static void ProcessArchivalReport(void); static void WalRcvComputeNextWakeup(WalRcvWakeupReason reason, TimestampTz now); @@ -875,6 +881,30 @@ XLogWalRcvProcessMsg(unsigned char type, char *buf, Size len, TimeLineID tli) XLogWalRcvSendReply(true, false); break; } + case PqReplMsg_ArchiveStatusReport: + { + /* Check that the filename looks valid */ + if (len >= sizeof(primary_last_archived)) + ereport(ERROR, + (errcode(ERRCODE_PROTOCOL_VIOLATION), + errmsg_internal("invalid archival report message with length %d", + (int) len))); + + memcpy(primary_last_archived, buf, len); + primary_last_archived[len] = '\0'; + + /* Verify it contains only valid characters */ + if (strspn(buf, VALID_XFN_CHARS) != len) + { + primary_last_archived[0] = '\0'; + ereport(ERROR, + (errcode(ERRCODE_PROTOCOL_VIOLATION), + errmsg_internal("unexpected character in primary's last archived filename"))); + } + + ProcessArchivalReport(); + break; + } default: ereport(ERROR, (errcode(ERRCODE_PROTOCOL_VIOLATION), @@ -1065,12 +1095,39 @@ XLogWalRcvClose(XLogRecPtr recptr, TimeLineID tli) /* * Create .done file forcibly to prevent the streamed segment from being - * archived later. + * archived later, unless archive_mode is 'always' or 'shared'. + * + * In 'always' mode, the standby archives independently. + * + * In 'shared' mode, we optimize by checking if this segment is already + * covered by the last archival report from the primary. If so, create + * .done directly. Otherwise, create .ready and wait for the next report. */ - if (XLogArchiveMode != ARCHIVE_MODE_ALWAYS) - XLogArchiveForceDone(xlogfname); - else + if (XLogArchiveMode == ARCHIVE_MODE_ALWAYS) + { XLogArchiveNotify(xlogfname); + } + else if (XLogArchiveMode == ARCHIVE_MODE_SHARED) + { + /* + * In shared mode, check if this segment is already archived on primary. + * If we're on the same timeline and this segment is <= last archived, + * mark it .done immediately. Otherwise create .ready. + */ + if (primary_last_archived_tli == recvFileTLI && + recvSegNo <= primary_last_archived_segno) + { + XLogArchiveForceDone(xlogfname); + } + else + { + XLogArchiveNotify(xlogfname); + } + } + else + { + XLogArchiveForceDone(xlogfname); + } recvFile = -1; } @@ -1247,6 +1304,87 @@ XLogWalRcvSendHSFeedback(bool immed) primary_has_standby_xmin = false; } +/* + * Process archival report from primary. + * + * The primary sends us the last WAL segment it has archived. We scan the + * archive_status directory for .ready files and mark segments on the same + * timeline as .done if they're <= the reported segment. + */ +static void +ProcessArchivalReport(void) +{ + TimeLineID reported_tli; + XLogSegNo reported_segno; + DIR *status_dir; + struct dirent *status_de; + char status_path[MAXPGPATH]; + + elog(DEBUG2, "received archival report from primary: %s", + primary_last_archived); + + /* Parse the reported WAL filename */ + if (!IsXLogFileName(primary_last_archived)) + { + elog(DEBUG2, "invalid WAL filename in archival report: %s", + primary_last_archived); + return; + } + + XLogFromFileName(primary_last_archived, &reported_tli, &reported_segno, + wal_segment_size); + + /* Remember the last archived segment for XLogWalRcvClose() */ + primary_last_archived_tli = reported_tli; + primary_last_archived_segno = reported_segno; + + /* Scan archive_status directory for .ready files */ + snprintf(status_path, MAXPGPATH, XLOGDIR "/archive_status"); + status_dir = AllocateDir(status_path); + if (status_dir == NULL) + { + elog(DEBUG2, "could not open archive_status directory: %m"); + return; + } + + while ((status_de = ReadDir(status_dir, status_path)) != NULL) + { + char *ready_suffix; + char walfile[MAXPGPATH]; + TimeLineID file_tli; + XLogSegNo file_segno; + /* Look for .ready files only */ + ready_suffix = strstr(status_de->d_name, ".ready"); + if (ready_suffix == NULL || ready_suffix[6] != '\0') + continue; + + /* Extract WAL filename (remove .ready suffix) */ + strlcpy(walfile, status_de->d_name, ready_suffix - status_de->d_name + 1); + + /* Parse the WAL filename */ + if (!IsXLogFileName(walfile)) + continue; + + XLogFromFileName(walfile, &file_tli, &file_segno, wal_segment_size); + + /* + * Mark as .done if it's on the same timeline and not after the + * reported segment. We only process the reported timeline to avoid + * marking segments from parent or future timelines prematurely. + * XXX: Process possible TLI switches happened between status reports. + * For now, leave segments on previous TLIs to archive_command. + */ + if (file_tli == reported_tli && file_segno <= reported_segno) + { + XLogArchiveForceDone(walfile); + elog(DEBUG3, "marked WAL segment %s as archived (primary archived up to %s)", + walfile, primary_last_archived); + } + } + + FreeDir(status_dir); +} + /* * Update shared memory status upon receiving a message from primary. * diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 3d8efbb69cc..c3472521cb2 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -193,6 +193,17 @@ static TimestampTz last_reply_timestamp = 0; /* Have we sent a heartbeat message asking for reply, since last reply? */ static bool waiting_for_ping_response = false; +/* + * Last archived WAL file. This is fetched from pgstat periodically and sent + * to the standby. last_archival_report_timestamp tracks when we last sent + * the report to avoid excessive pgstat access. + */ +static char last_archived_wal[MAX_XFN_CHARS + 1]; +static TimestampTz last_archival_report_timestamp = 0; + +/* Interval for sending archival reports (10 seconds) */ +#define ARCHIVAL_REPORT_INTERVAL 10000 + /* * While streaming WAL in Copy mode, streamingDoneSending is set to true * after we have sent CopyDone. We should not send any more CopyData messages @@ -293,6 +304,7 @@ static void StartLogicalReplication(StartReplicationCmd *cmd); static void ProcessStandbyMessage(void); static void ProcessStandbyReplyMessage(void); static void ProcessStandbyHSFeedbackMessage(void); +static void WalSndArchivalReport(void); static void ProcessRepliesIfAny(void); static void ProcessPendingWrites(void); static void WalSndKeepalive(bool requestReply, XLogRecPtr writePtr); @@ -2752,6 +2764,84 @@ ProcessStandbyHSFeedbackMessage(void) } } +/* + * Send archival status report to standby. + * + * This is called periodically during physical replication to inform the + * standby about the last WAL segment archived by the primary. The standby + * can then mark segments up to that point as .done, allowing them to be + * recycled. This prevents WAL loss during standby promotion. + */ +static void +WalSndArchivalReport(void) +{ + PgStat_ArchiverStats *archiver_stats; + TimestampTz now; + char *last_archived; + + /* Only send reports when archive_mode=shared */ + if (XLogArchiveMode != ARCHIVE_MODE_SHARED) + return; + + /* Only send reports during physical streaming replication, not during backup */ + if (MyWalSnd->kind != REPLICATION_KIND_PHYSICAL) + return; + if (MyWalSnd->state != WALSNDSTATE_CATCHUP && + MyWalSnd->state != WALSNDSTATE_STREAMING) + return; + + /* + * Don't send to temporary replication slots (used by pg_basebackup). + * Connections without slots (regular standbys) are OK. + */ + if (MyReplicationSlot != NULL && + MyReplicationSlot->data.persistency == RS_TEMPORARY) + return; + + now = GetCurrentTimestamp(); + + /* + * Send report at most once per ARCHIVAL_REPORT_INTERVAL (10 seconds). + * This avoids excessive pgstat access. + */ + if (now < TimestampTzPlusMilliseconds(last_archival_report_timestamp, + ARCHIVAL_REPORT_INTERVAL)) + return; + last_archival_report_timestamp = now; + /* + * Get archiver statistics. We use non-blocking access to avoid delaying + * replication if stats collector is slow. If stats are unavailable or + * stale, we'll just try again at the next interval. + */ + archiver_stats = pgstat_fetch_stat_archiver(); + if (archiver_stats == NULL) + return; + + last_archived = archiver_stats->last_archived_wal; + /* + * Only send a report if the last archived WAL has changed. This is both + * an optimization and ensures we don't send empty reports on startup. + */ + if (strcmp(last_archived, last_archived_wal) == 0) + return; + + /* Only send reports for WAL segments, not backup history files or other archived files */ + if (!IsXLogFileName(last_archived)) + return; + + elog(DEBUG2, "sending archival report: %s", last_archived); + + /* Remember what we sent */ + strlcpy(last_archived_wal, last_archived, sizeof(last_archived_wal)); + + /* Construct the message... */ + resetStringInfo(&output_message); + pq_sendbyte(&output_message, PqReplMsg_ArchiveStatusReport); + pq_sendbytes(&output_message, last_archived, strlen(last_archived)); + /* ... and send it wrapped in CopyData */ + pq_putmessage_noblock(PqMsg_CopyData, output_message.data, output_message.len); +} + /* * Compute how long send/receive loops should sleep. * @@ -4171,6 +4261,9 @@ WalSndKeepaliveIfNecessary(void) if (pq_flush_if_writable() != 0) WalSndShutdown(); } + + /* Send archival status report if needed */ + WalSndArchivalReport(); } /* diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h index f20f5edb438..e9e6973e117 100644 --- a/src/include/access/xlog.h +++ b/src/include/access/xlog.h @@ -65,6 +65,7 @@ typedef enum ArchiveMode ARCHIVE_MODE_OFF = 0, /* disabled */ ARCHIVE_MODE_ON, /* enabled while server is running normally */ ARCHIVE_MODE_ALWAYS, /* enabled always (even during recovery) */ + ARCHIVE_MODE_SHARED, /* shared archive between primary and standby */ } ArchiveMode; extern PGDLLIMPORT int XLogArchiveMode; diff --git a/src/include/libpq/protocol.h b/src/include/libpq/protocol.h index b0bcb3cdc26..925c7568ea1 100644 --- a/src/include/libpq/protocol.h +++ b/src/include/libpq/protocol.h @@ -69,6 +69,28 @@ #define PqMsg_Progress 'P' +/* Replication codes sent by the primary (wrapped in CopyData messages). */ + +#define PqReplMsg_ArchiveStatusReport 'a' +#define PqReplMsg_Keepalive 'k' +#define PqReplMsg_PrimaryStatusUpdate 's' +#define PqReplMsg_WALData 'w' + + +/* Replication codes sent by the standby (wrapped in CopyData messages). */ + +#define PqReplMsg_HotStandbyFeedback 'h' +#define PqReplMsg_PrimaryStatusRequest 'p' +#define PqReplMsg_StandbyStatusUpdate 'r' + + +/* Codes used for backups via COPY OUT (wrapped in CopyData messages). */ + +#define PqBackupMsg_Manifest 'm' +#define PqBackupMsg_NewArchive 'n' +#define PqBackupMsg_ProgressReport 'p' + + /* These are the authentication request codes sent by the backend. */ #define AUTH_REQ_OK 0 /* User is authenticated */ diff --git a/src/test/recovery/t/050_archive_shared.pl b/src/test/recovery/t/050_archive_shared.pl new file mode 100644 index 00000000000..397b71ad79d --- /dev/null +++ b/src/test/recovery/t/050_archive_shared.pl @@ -0,0 +1,270 @@ +# Copyright (c) 2025, PostgreSQL Global Development Group + +# Test archive_mode=shared for coordinated WAL archiving between primary and standby +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; +use File::Path qw(rmtree); + +# Initialize primary node with archiving +my $archive_dir = PostgreSQL::Test::Utils::tempdir(); +my $primary = PostgreSQL::Test::Cluster->new('primary'); +$primary->init(has_archiving => 1, allows_streaming => 1); +$primary->append_conf('postgresql.conf', " +archive_mode = shared +archive_command = 'cp %p \"$archive_dir\"/%f' +wal_keep_size = 128MB +"); +$primary->start; + +# Create a test table and generate some WAL +$primary->safe_psql('postgres', 'CREATE TABLE test_table (id int, data text);'); +$primary->safe_psql('postgres', "INSERT INTO test_table SELECT i, 'data' || i FROM generate_series(1, 500) i;"); +$primary->safe_psql('postgres', 'SELECT pg_switch_wal();'); +$primary->safe_psql('postgres', "INSERT INTO test_table SELECT i, 'data' || i FROM generate_series(501, 1000) i;"); +$primary->safe_psql('postgres', 'SELECT pg_switch_wal();'); + +# Wait for archiver to archive segments +$primary->poll_query_until('postgres', + "SELECT archived_count > 0 FROM pg_stat_archiver") + or die "Timed out waiting for archiver to start"; + +my $archived_count = () = glob("$archive_dir/*"); +ok($archived_count > 0, "primary has archived WAL files to shared archive"); +note("Primary archived $archived_count files"); + +# Take backup for standby +my $backup_name = 'standby_backup'; +$primary->backup($backup_name); + +# Exclude possible race condition when backup WAL is last archived +$primary->safe_psql('postgres', "INSERT INTO test_table SELECT i, 'data' || i FROM generate_series(501, 1000) i;"); +$primary->safe_psql('postgres', 'SELECT pg_switch_wal();'); + +# Set up standby with archive_mode=shared +my $standby = PostgreSQL::Test::Cluster->new('standby'); +$standby->init_from_backup($primary, $backup_name, has_streaming => 1); +$standby->append_conf('postgresql.conf', " +archive_mode = shared +archive_command = 'cp %p \"$archive_dir\"/%f' +wal_receiver_status_interval = 1s +"); +$standby->start; + +# Wait for standby to catch up +$primary->wait_for_catchup($standby); + +# Generate more WAL on primary (these are new segments not yet archived) +$primary->safe_psql('postgres', "INSERT INTO test_table SELECT i, 'data' || i FROM generate_series(1001, 1500) i;"); +$primary->safe_psql('postgres', 'SELECT pg_switch_wal();'); +$primary->safe_psql('postgres', "INSERT INTO test_table SELECT i, 'data' || i FROM generate_series(1501, 2000) i;"); +$primary->safe_psql('postgres', 'SELECT pg_switch_wal();'); + +# Wait for standby to receive the new WAL +$primary->wait_for_catchup($standby); + +# Check that standby has .ready or .done files for the newly received segments. +# Normally they should be .ready (not yet archived by primary), but in rare cases +# the archiver could be very fast and an archive report sent immediately, creating +# .done files instead. Both are correct behavior - the key is that files exist. +my $standby_archive_status = $standby->data_dir . '/pg_wal/archive_status'; +my $status_count = 0; +if (opendir(my $dh, $standby_archive_status)) +{ + my @files = grep { /\.(ready|done)$/ } readdir($dh); + $status_count = scalar(@files); + my $ready_count = scalar(grep { /\.ready$/ } @files); + my $done_count = scalar(grep { /\.done$/ } @files); + note("Standby has $ready_count .ready files and $done_count .done files"); + closedir($dh); +} +cmp_ok($status_count, '>', 0, "standby creates archive status files for received WAL"); + +# Generate more WAL and wait for archiving on primary +my $initial_archived = $primary->safe_psql('postgres', 'SELECT archived_count FROM pg_stat_archiver'); +$primary->safe_psql('postgres', "INSERT INTO test_table SELECT i, 'more-data' || i FROM generate_series(2001, 2500) i;"); +$primary->safe_psql('postgres', 'SELECT pg_switch_wal();'); +$primary->safe_psql('postgres', "INSERT INTO test_table SELECT i, 'more-data2' || i FROM generate_series(2501, 3000) i;"); +$primary->safe_psql('postgres', 'SELECT pg_switch_wal();'); + +# Wait for primary to archive the new segments +$primary->poll_query_until('postgres', + "SELECT archived_count > $initial_archived FROM pg_stat_archiver") + or die "Timed out waiting for primary to archive new segments"; + +# Wait for standby to catch up (archive status is sent during replication) +$primary->wait_for_catchup($standby); + +# Wait for primary to send archival status updates and standby to process them +# The standby should mark segments as .done after receiving archive status from primary +my $done_count = 0; +for (my $i = 0; $i < $PostgreSQL::Test::Utils::timeout_default; $i++) +{ + $done_count = 0; + if (opendir(my $dh, $standby_archive_status)) + { + $done_count = scalar(grep { /\.done$/ } readdir($dh)); + closedir($dh); + } + last if $done_count > 0; + sleep(1); +} +ok($done_count > 0, "standby marked segments as .done after primary's archival report"); +note("Standby has $done_count .done files"); + +############################################################################### +# Test 2: Standby promotion - verify archiver activates +############################################################################### + +# Before promotion, verify archiver is not running on standby (shared mode during recovery) +# In shared mode, the standby's archiver should not be archiving during recovery +my $archived_before = $standby->safe_psql('postgres', + "SELECT archived_count FROM pg_stat_archiver"); +is($archived_before, '0', + "archiver not active on standby before promotion (archived_count=0)"); + +# Verify standby is still in recovery before promoting +my $in_recovery = $standby->safe_psql('postgres', "SELECT pg_is_in_recovery();"); +is($in_recovery, 't', "standby is in recovery before promotion"); + +# Promote the standby +$standby->promote; +$standby->poll_query_until('postgres', "SELECT NOT pg_is_in_recovery();"); + +# Generate WAL on new primary (former standby) +$standby->safe_psql('postgres', "INSERT INTO test_table SELECT i, 'post-promotion' || i FROM generate_series(2001, 2500) i;"); +$standby->safe_psql('postgres', 'SELECT pg_switch_wal();'); + +# Wait for archiver to activate and archive the new WAL +# Check pg_stat_archiver to verify archiving is happening +$standby->poll_query_until('postgres', + "SELECT archived_count > 0 FROM pg_stat_archiver") + or die "Timed out waiting for promoted standby to start archiving"; +pass("promoted standby started archiving"); + +# Verify data integrity +my $count = $standby->safe_psql('postgres', 'SELECT COUNT(*) FROM test_table;'); +ok($count >= 2500, "promoted standby has all data (got $count rows)"); + +############################################################################### +# Test 3: Cascading replication +############################################################################### + +# Take a backup from the promoted standby (now the new primary) +my $promoted_backup = 'promoted_backup'; +$standby->backup($promoted_backup); + +# Set up second-level standby (cascading from first standby, now promoted) +my $standby2 = PostgreSQL::Test::Cluster->new('standby2'); +$standby2->init_from_backup($standby, $promoted_backup, has_streaming => 1); +$standby2->append_conf('postgresql.conf', " +archive_mode = shared +archive_command = 'cp %p \"$archive_dir\"/%f' +wal_receiver_status_interval = 1s +"); +$standby2->start; + +# Generate WAL on promoted standby (now primary for standby2) +my $cascading_archived_before = $standby->safe_psql('postgres', 'SELECT archived_count FROM pg_stat_archiver'); +$standby->safe_psql('postgres', "INSERT INTO test_table SELECT i, 'cascading' || i FROM generate_series(2501, 3000) i;"); +$standby->safe_psql('postgres', 'SELECT pg_switch_wal();'); + +# Wait for the promoted standby (acting as primary) to archive the new segment +$standby->poll_query_until('postgres', + "SELECT archived_count > $cascading_archived_before FROM pg_stat_archiver") + or die "Timed out waiting for primary to archive segment in cascading test"; + +# Wait for cascading standby to catch up +$standby->wait_for_catchup($standby2); + +# Wait for cascading standby to receive archive status and mark segments as .done +my $standby2_archive_status = $standby2->data_dir . '/pg_wal/archive_status'; +my $standby2_done_count = 0; +for (my $i = 0; $i < $PostgreSQL::Test::Utils::timeout_default; $i++) +{ + $standby2_done_count = 0; + if (opendir(my $dh, $standby2_archive_status)) + { + $standby2_done_count = scalar(grep { /\.done$/ } readdir($dh)); + closedir($dh); + } + last if $standby2_done_count > 0; + sleep(1); +} +ok($standby2_done_count > 0, "cascading standby marks segments as .done"); +note("Cascading standby has $standby2_done_count .done files"); + +# Verify cascading standby has all data +my $standby2_count = $standby2->safe_psql('postgres', 'SELECT COUNT(*) FROM test_table;'); +ok($standby2_count >= 3000, "cascading standby has all data (got $standby2_count rows)"); + +############################################################################### +# Test 4: Multiple standbys from same primary +############################################################################### + +# Create third standby from promoted standby (current primary) +my $standby3 = PostgreSQL::Test::Cluster->new('standby3'); +my $backup2 = 'multi_standby_backup'; +$standby->backup($backup2); +$standby3->init_from_backup($standby, $backup2, has_streaming => 1); +$standby3->append_conf('postgresql.conf', " +archive_mode = shared +archive_command = 'cp %p \"$archive_dir\"/%f' +wal_receiver_status_interval = 1s +"); +$standby3->start; + +# Generate WAL and ensure both standbys receive it +my $standby_archived_before = $standby->safe_psql('postgres', 'SELECT archived_count FROM pg_stat_archiver'); +$standby->safe_psql('postgres', "INSERT INTO test_table SELECT i, 'multi' || i FROM generate_series(3001, 3500) i;"); +$standby->safe_psql('postgres', 'SELECT pg_switch_wal();'); + +# Wait for the promoted standby (acting as primary) to archive the new segment +$standby->poll_query_until('postgres', + "SELECT archived_count > $standby_archived_before FROM pg_stat_archiver") + or die "Timed out waiting for primary to archive segment in multi-standby test"; + +$standby->wait_for_catchup($standby2); +$standby->wait_for_catchup($standby3); + +# Verify both standbys eventually mark segments as .done +my $standby3_archive_status = $standby3->data_dir . '/pg_wal/archive_status'; + +for (my $i = 0; $i < $PostgreSQL::Test::Utils::timeout_default; $i++) +{ + $standby2_done_count = 0; + if (opendir(my $dh, $standby2_archive_status)) + { + $standby2_done_count = scalar(grep { /\.done$/ } readdir($dh)); + closedir($dh); + } + last if $standby2_done_count > 0; + sleep(1); +} + +my $standby3_done_count = 0; +for (my $i = 0; $i < $PostgreSQL::Test::Utils::timeout_default; $i++) +{ + $standby3_done_count = 0; + if (opendir(my $dh, $standby3_archive_status)) + { + $standby3_done_count = scalar(grep { /\.done$/ } readdir($dh)); + closedir($dh); + } + last if $standby3_done_count > 0; + sleep(1); +} + +ok($standby2_done_count > 0, "standby2 marks segments as .done"); +ok($standby3_done_count > 0, "standby3 marks segments as .done"); +note("standby2 has $standby2_done_count .done files, standby3 has $standby3_done_count .done files"); + +# Verify both standbys have all data +$standby2_count = $standby2->safe_psql('postgres', 'SELECT COUNT(*) FROM test_table;'); +my $standby3_count = $standby3->safe_psql('postgres', 'SELECT COUNT(*) FROM test_table;'); +ok($standby2_count >= 3500, "standby2 has all data (got $standby2_count rows)"); +ok($standby3_count >= 3500, "standby3 has all data (got $standby3_count rows)"); + +done_testing(); From 505d0d59a6896fa9f3ba0df74adae0c29f3d01d3 Mon Sep 17 00:00:00 2001 From: Andrey Borodin Date: Tue, 10 Feb 2026 16:45:10 +0500 Subject: [PATCH 40/45] Mark ancestor timeline WAL segments as archived When standby receives archive status report, check if .ready files belong to ancestor timelines before the switch point and mark them as .done if already archived by primary. --- src/backend/replication/walreceiver.c | 55 ++++++++++++++++++++++++--- 1 file changed, 50 insertions(+), 5 deletions(-) diff --git a/src/backend/replication/walreceiver.c b/src/backend/replication/walreceiver.c index 524b40cdeb4..b241e942204 100644 --- a/src/backend/replication/walreceiver.c +++ b/src/backend/replication/walreceiver.c @@ -1113,6 +1113,11 @@ XLogWalRcvClose(XLogRecPtr recptr, TimeLineID tli) * In shared mode, check if this segment is already archived on primary. * If we're on the same timeline and this segment is <= last archived, * mark it .done immediately. Otherwise create .ready. + * + * We don't check ancestor timeline cases here to avoid reading timeline + * history files on every segment close. ProcessArchivalReport() will + * handle marking ancestor timeline segments as .done when it scans + * the archive_status directory. */ if (primary_last_archived_tli == recvFileTLI && recvSegNo <= primary_last_archived_segno) @@ -1319,6 +1324,7 @@ ProcessArchivalReport(void) DIR *status_dir; struct dirent *status_de; char status_path[MAXPGPATH]; + List *tli_history = NIL; elog(DEBUG2, "received archival report from primary: %s", primary_last_archived); @@ -1368,18 +1374,57 @@ ProcessArchivalReport(void) XLogFromFileName(walfile, &file_tli, &file_segno, wal_segment_size); /* - * Mark as .done if it's on the same timeline and not after the - * reported segment. We only process the reported timeline to avoid - * marking segments from parent or future timelines prematurely. - * XXX: Process possible TLI switches happened between status reports. - * For now, leave segments on previous TLIs to archive_command. + * Mark as .done if: + * 1. Same timeline and segment <= reported segment, OR + * 2. Ancestor timeline and segment is before the timeline switch point + * + * For ancestor timelines: if primary archived segment X on timeline T, + * then all segments on ancestor timelines before the switch to T must + * have been archived (they're required to reach timeline T). */ if (file_tli == reported_tli && file_segno <= reported_segno) { + /* Same timeline, segment already archived */ XLogArchiveForceDone(walfile); elog(DEBUG3, "marked WAL segment %s as archived (primary archived up to %s)", walfile, primary_last_archived); } + else if (file_tli != reported_tli) + { + /* + * Different timeline - check if it's an ancestor and if this + * segment is before the timeline switch point. Only read timeline + * history if we haven't already (lazy loading). + * + * Note: Timelines form a tree structure, not a linear sequence, + * so we can't use < or > to compare them. + */ + if (tli_history == NIL) + tli_history = readTimeLineHistory(reported_tli); + + if (tliInHistory(file_tli, tli_history)) + { + XLogRecPtr switchpoint; + XLogSegNo switchpoint_segno; + + /* Get the point where we switched away from this timeline */ + switchpoint = tliSwitchPoint(file_tli, tli_history, NULL); + + /* + * If the segment is at or before the switch point, it must have + * been archived (it's required to reach the reported timeline). + * The segment containing the switch point belongs to the old + * timeline up to the switch point and should be archived. + */ + XLByteToSeg(switchpoint, switchpoint_segno, wal_segment_size); + if (file_segno <= switchpoint_segno) + { + XLogArchiveForceDone(walfile); + elog(DEBUG3, "marked ancestor timeline segment %s as archived (before switch to timeline %u)", + walfile, reported_tli); + } + } + } } FreeDir(status_dir); From 052740b640da85c65adeab3b3308f586230a9061 Mon Sep 17 00:00:00 2001 From: Andrey Borodin Date: Wed, 11 Feb 2026 18:17:25 +0500 Subject: [PATCH 41/45] Optimize ProcessArchivalReport to avoid directory scans When archive status reports arrive sequentially on the same timeline, directly generate expected WAL filenames and mark them as archived instead of scanning the entire archive_status directory. This optimization reduces overhead in the common case where the primary continuously archives segments. Directory scan is still used when: - Timeline changes (to handle ancestor timelines) - First report received - Non-sequential reports XLogArchiveForceDone() handles all cases internally (checking if .done exists, if .ready exists, or creating .done if neither exists), so no pre-check is needed. --- src/backend/replication/walreceiver.c | 196 +++++++++++++++++--------- 1 file changed, 132 insertions(+), 64 deletions(-) diff --git a/src/backend/replication/walreceiver.c b/src/backend/replication/walreceiver.c index b241e942204..f985dd0f5cb 100644 --- a/src/backend/replication/walreceiver.c +++ b/src/backend/replication/walreceiver.c @@ -136,6 +136,14 @@ static char primary_last_archived[MAX_XFN_CHARS + 1]; static TimeLineID primary_last_archived_tli = 0; static XLogSegNo primary_last_archived_segno = 0; +/* + * Last segment we successfully marked as .done. Used to optimize + * ProcessArchivalReport() by generating expected filenames instead + * of scanning the archive_status directory. + */ +static TimeLineID last_processed_tli = 0; +static XLogSegNo last_processed_segno = 0; + /* Prototypes for private functions */ static void WalRcvFetchTimeLineHistoryFiles(TimeLineID first, TimeLineID last); static void WalRcvWaitForStartPosition(XLogRecPtr *startpoint, TimeLineID *startpointTLI); @@ -1321,10 +1329,9 @@ ProcessArchivalReport(void) { TimeLineID reported_tli; XLogSegNo reported_segno; - DIR *status_dir; - struct dirent *status_de; char status_path[MAXPGPATH]; - List *tli_history = NIL; + bool use_direct_check = false; + XLogSegNo start_segno; elog(DEBUG2, "received archival report from primary: %s", primary_last_archived); @@ -1344,90 +1351,151 @@ ProcessArchivalReport(void) primary_last_archived_tli = reported_tli; primary_last_archived_segno = reported_segno; - /* Scan archive_status directory for .ready files */ - snprintf(status_path, MAXPGPATH, XLOGDIR "/archive_status"); - status_dir = AllocateDir(status_path); - if (status_dir == NULL) + /* + * Optimization: If the new report is on the same timeline as the last + * processed segment and moves forward, we can directly check for .ready + * files for segments between last_processed_segno and reported_segno + * instead of scanning the entire archive_status directory. + * + * Fall back to directory scan if: + * - Timeline changed (need to handle ancestor timelines) + * - This is the first report (last_processed_tli == 0) + * - Reported segment is not ahead (nothing new to process) + */ + if (last_processed_tli == reported_tli && + last_processed_tli != 0 && + reported_segno > last_processed_segno) { - elog(DEBUG2, "could not open archive_status directory: %m"); - return; + use_direct_check = true; + start_segno = last_processed_segno + 1; } - while ((status_de = ReadDir(status_dir, status_path)) != NULL) + if (use_direct_check) { - char *ready_suffix; - char walfile[MAXPGPATH]; - TimeLineID file_tli; - XLogSegNo file_segno; - /* Look for .ready files only */ - ready_suffix = strstr(status_de->d_name, ".ready"); - if (ready_suffix == NULL || ready_suffix[6] != '\0') - continue; - - /* Extract WAL filename (remove .ready suffix) */ - strlcpy(walfile, status_de->d_name, ready_suffix - status_de->d_name + 1); - - /* Parse the WAL filename */ - if (!IsXLogFileName(walfile)) - continue; - - XLogFromFileName(walfile, &file_tli, &file_segno, wal_segment_size); - /* - * Mark as .done if: - * 1. Same timeline and segment <= reported segment, OR - * 2. Ancestor timeline and segment is before the timeline switch point - * - * For ancestor timelines: if primary archived segment X on timeline T, - * then all segments on ancestor timelines before the switch to T must - * have been archived (they're required to reach timeline T). + * Direct check: generate filenames for expected segments. + * XLogArchiveForceDone() will handle the case where .ready doesn't + * exist or .done already exists, so no need to stat() first. */ - if (file_tli == reported_tli && file_segno <= reported_segno) + XLogSegNo segno; + + for (segno = start_segno; segno <= reported_segno; segno++) { - /* Same timeline, segment already archived */ + char walfile[MAXFNAMELEN]; + + /* Generate WAL filename and mark as archived */ + XLogFileName(walfile, reported_tli, segno, wal_segment_size); XLogArchiveForceDone(walfile); elog(DEBUG3, "marked WAL segment %s as archived (primary archived up to %s)", walfile, primary_last_archived); + + /* Track the last segment we processed */ + last_processed_tli = reported_tli; + last_processed_segno = segno; + } + } + else + { + /* + * Directory scan: needed when timeline changed or first report. + * This handles both same-timeline and ancestor-timeline cases. + */ + DIR *status_dir; + struct dirent *status_de; + List *tli_history = NIL; + + snprintf(status_path, MAXPGPATH, XLOGDIR "/archive_status"); + status_dir = AllocateDir(status_path); + if (status_dir == NULL) + { + elog(DEBUG2, "could not open archive_status directory: %m"); + return; } - else if (file_tli != reported_tli) + + while ((status_de = ReadDir(status_dir, status_path)) != NULL) { + char *ready_suffix; + char walfile[MAXPGPATH]; + TimeLineID file_tli; + XLogSegNo file_segno; + + /* Look for .ready files only */ + ready_suffix = strstr(status_de->d_name, ".ready"); + if (ready_suffix == NULL || ready_suffix[6] != '\0') + continue; + + /* Extract WAL filename (remove .ready suffix) */ + strlcpy(walfile, status_de->d_name, ready_suffix - status_de->d_name + 1); + + /* Parse the WAL filename */ + if (!IsXLogFileName(walfile)) + continue; + + XLogFromFileName(walfile, &file_tli, &file_segno, wal_segment_size); + /* - * Different timeline - check if it's an ancestor and if this - * segment is before the timeline switch point. Only read timeline - * history if we haven't already (lazy loading). + * Mark as .done if: + * 1. Same timeline and segment <= reported segment, OR + * 2. Ancestor timeline and segment is before the timeline switch point * - * Note: Timelines form a tree structure, not a linear sequence, - * so we can't use < or > to compare them. + * For ancestor timelines: if primary archived segment X on timeline T, + * then all segments on ancestor timelines before the switch to T must + * have been archived (they're required to reach timeline T). */ - if (tli_history == NIL) - tli_history = readTimeLineHistory(reported_tli); - - if (tliInHistory(file_tli, tli_history)) + if (file_tli == reported_tli && file_segno <= reported_segno) + { + /* Same timeline, segment already archived */ + XLogArchiveForceDone(walfile); + elog(DEBUG3, "marked WAL segment %s as archived (primary archived up to %s)", + walfile, primary_last_archived); + } + else if (file_tli != reported_tli) { - XLogRecPtr switchpoint; - XLogSegNo switchpoint_segno; - - /* Get the point where we switched away from this timeline */ - switchpoint = tliSwitchPoint(file_tli, tli_history, NULL); - /* - * If the segment is at or before the switch point, it must have - * been archived (it's required to reach the reported timeline). - * The segment containing the switch point belongs to the old - * timeline up to the switch point and should be archived. + * Different timeline - check if it's an ancestor and if this + * segment is before the timeline switch point. Only read timeline + * history if we haven't already (lazy loading). + * + * Note: Timelines form a tree structure, not a linear sequence, + * so we can't use < or > to compare them. */ - XLByteToSeg(switchpoint, switchpoint_segno, wal_segment_size); - if (file_segno <= switchpoint_segno) + if (tli_history == NIL) + tli_history = readTimeLineHistory(reported_tli); + + if (tliInHistory(file_tli, tli_history)) { - XLogArchiveForceDone(walfile); - elog(DEBUG3, "marked ancestor timeline segment %s as archived (before switch to timeline %u)", - walfile, reported_tli); + XLogRecPtr switchpoint; + XLogSegNo switchpoint_segno; + + /* Get the point where we switched away from this timeline */ + switchpoint = tliSwitchPoint(file_tli, tli_history, NULL); + + /* + * If the segment is at or before the switch point, it must have + * been archived (it's required to reach the reported timeline). + * The segment containing the switch point belongs to the old + * timeline up to the switch point and should be archived. + */ + XLByteToSeg(switchpoint, switchpoint_segno, wal_segment_size); + if (file_segno <= switchpoint_segno) + { + XLogArchiveForceDone(walfile); + elog(DEBUG3, "marked ancestor timeline segment %s as archived (before switch to timeline %u)", + walfile, reported_tli); + } } } } - } - FreeDir(status_dir); + FreeDir(status_dir); + + /* + * After a full directory scan following a timeline change, update + * our tracking to the newly reported position for future optimizations. + */ + last_processed_tli = reported_tli; + last_processed_segno = reported_segno; + } } /* From be6ab296a8bbee8e917e84ec4cbd4bbbaea98d6f Mon Sep 17 00:00:00 2001 From: Andrey Borodin Date: Thu, 12 Feb 2026 22:25:09 +0500 Subject: [PATCH 42/45] Fuse shared archive with ycmdb.shared_archive --- src/backend/access/transam/xlog.c | 1 + src/backend/postmaster/pgarch.c | 2 +- src/backend/replication/walreceiver.c | 2 +- src/backend/replication/walsender.c | 5 +++-- src/backend/utils/misc/guc_tables.c | 12 ++++++++++++ src/include/access/xlog.h | 9 +++++++++ 6 files changed, 27 insertions(+), 4 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8e6b4e0470b..a99ee640a69 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -118,6 +118,7 @@ int wal_keep_size_mb = 0; int XLOGbuffers = -1; int XLogArchiveTimeout = 0; int XLogArchiveMode = ARCHIVE_MODE_OFF; +bool ycmdb_shared_archive = false; /* makes archive_mode=on act as shared */ char *XLogArchiveCommand = NULL; bool EnableHotStandby = false; bool fullPageWrites = true; diff --git a/src/backend/postmaster/pgarch.c b/src/backend/postmaster/pgarch.c index bb22da47580..5b0f9fcea96 100644 --- a/src/backend/postmaster/pgarch.c +++ b/src/backend/postmaster/pgarch.c @@ -388,7 +388,7 @@ pgarch_ArchiverCopyLoop(void) * marks files as .done when the primary confirms archival. After * promotion, the archiver starts working normally. */ - if (XLogArchiveMode == ARCHIVE_MODE_SHARED && RecoveryInProgress()) + if (EffectiveArchiveModeIsShared() && RecoveryInProgress()) return; /* force directory scan in the first call to pgarch_readyXlog() */ diff --git a/src/backend/replication/walreceiver.c b/src/backend/replication/walreceiver.c index f985dd0f5cb..125655fd72b 100644 --- a/src/backend/replication/walreceiver.c +++ b/src/backend/replication/walreceiver.c @@ -1115,7 +1115,7 @@ XLogWalRcvClose(XLogRecPtr recptr, TimeLineID tli) { XLogArchiveNotify(xlogfname); } - else if (XLogArchiveMode == ARCHIVE_MODE_SHARED) + else if (EffectiveArchiveModeIsShared()) { /* * In shared mode, check if this segment is already archived on primary. diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index c3472521cb2..b00ac57486f 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -84,6 +84,7 @@ #include "storage/ipc.h" #include "storage/pmsignal.h" #include "storage/proc.h" +#include "storage/procarray.h" #include "tcop/dest.h" #include "tcop/tcopprot.h" #include "utils/acl.h" @@ -2779,8 +2780,8 @@ WalSndArchivalReport(void) TimestampTz now; char *last_archived; - /* Only send reports when archive_mode=shared */ - if (XLogArchiveMode != ARCHIVE_MODE_SHARED) + /* Only send reports when shared archive is active */ + if (!EffectiveArchiveModeIsShared()) return; /* Only send reports during physical streaming replication, not during backup */ diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index 96f25e57e09..f4419f8acf5 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -34,6 +34,7 @@ #include "access/slru.h" #include "access/toast_compression.h" #include "access/twophase.h" +#include "access/xlog.h" #include "access/xlog_internal.h" #include "access/xlogprefetcher.h" #include "access/xlogrecovery.h" @@ -1226,6 +1227,17 @@ struct config_bool ConfigureNamesBool[] = NULL, NULL, NULL }, + { + {"ycmdb.shared_archive", PGC_POSTMASTER, WAL_ARCHIVING, + gettext_noop("Makes archive_mode=on behave as shared (for managed service compatibility)."), + gettext_noop("When true, archive_mode=on is treated as archive_mode=shared. Does not affect archive_mode=off or archive_mode=always. Used when control plane cannot configure archive_mode=shared directly."), + GUC_NOT_IN_SAMPLE + }, + &ycmdb_shared_archive, + false, + NULL, NULL, NULL + }, + { {"wal_init_zero", PGC_SUSET, WAL_SETTINGS, gettext_noop("Writes zeroes to new WAL files before first use."), diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h index e9e6973e117..3aa18a584d0 100644 --- a/src/include/access/xlog.h +++ b/src/include/access/xlog.h @@ -68,6 +68,15 @@ typedef enum ArchiveMode ARCHIVE_MODE_SHARED, /* shared archive between primary and standby */ } ArchiveMode; extern PGDLLIMPORT int XLogArchiveMode; +extern PGDLLIMPORT bool ycmdb_shared_archive; + +/* + * True when shared archive behavior is active: either archive_mode=shared + * or archive_mode=on with ycmdb.shared_archive=true (managed service). + */ +#define EffectiveArchiveModeIsShared() \ + (XLogArchiveMode == ARCHIVE_MODE_SHARED || \ + (XLogArchiveMode == ARCHIVE_MODE_ON && ycmdb_shared_archive)) /* WAL levels */ typedef enum WalLevel From 381e736ad058575088085ff544d77bffa8c2beeb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lvaro=20Herrera?= Date: Thu, 1 Jan 2026 13:47:51 -0300 Subject: [PATCH 43/45] REASSIGN OWNED: ignore subscriptions in other databases XXX do we also need DROP OWNED to work this way? Probably not. Discussion: https://postgr.es/m/CALdSSPhjONb+EftRD=J1pqajkB+pjT0=tbMJs16C6q9+xT8NNg@mail.gmail.com --- src/backend/catalog/pg_shdepend.c | 10 ++++++++++ src/backend/utils/cache/lsyscache.c | 22 ++++++++++++++++++++++ src/include/utils/lsyscache.h | 1 + 3 files changed, 33 insertions(+) diff --git a/src/backend/catalog/pg_shdepend.c b/src/backend/catalog/pg_shdepend.c index 536191284e8..cdcd91b7795 100644 --- a/src/backend/catalog/pg_shdepend.c +++ b/src/backend/catalog/pg_shdepend.c @@ -63,6 +63,7 @@ #include "utils/fmgroids.h" #include "utils/memutils.h" #include "utils/syscache.h" +#include "utils/lsyscache.h" typedef enum { @@ -1592,6 +1593,15 @@ shdepReassignOwned(List *roleids, Oid newrole) sdepForm->dbid != InvalidOid) continue; + /* + * Subscriptions are linked to specific databases, even though + * they are nominally shared objects. Skip those that aren't + * in this database. + */ + if (sdepForm->classid == SubscriptionRelationId && + get_subscription_database(sdepForm->objid) != MyDatabaseId) + continue; + /* * The various DDL routines called here tend to leak memory in * CurrentMemoryContext. That's not a problem when they're only diff --git a/src/backend/utils/cache/lsyscache.c b/src/backend/utils/cache/lsyscache.c index c460a72b75d..b5d4b7d06c6 100644 --- a/src/backend/utils/cache/lsyscache.c +++ b/src/backend/utils/cache/lsyscache.c @@ -3854,3 +3854,25 @@ get_subscription_name(Oid subid, bool missing_ok) return subname; } + +/* + * Return the OID of the database the given subscription is in. + */ +Oid +get_subscription_database(Oid subid) +{ + HeapTuple tup; + Form_pg_subscription subform; + Oid subdbid; + + tup = SearchSysCache1(SUBSCRIPTIONOID, ObjectIdGetDatum(subid)); + if (!HeapTupleIsValid(tup)) + elog(ERROR, "cache lookup failed for subscription %u", subid); + + subform = (Form_pg_subscription) GETSTRUCT(tup); + subdbid = subform->subdbid; + + ReleaseSysCache(tup); + + return subdbid; +} diff --git a/src/include/utils/lsyscache.h b/src/include/utils/lsyscache.h index fa7c7e0323b..e9f1ca9901e 100644 --- a/src/include/utils/lsyscache.h +++ b/src/include/utils/lsyscache.h @@ -210,6 +210,7 @@ extern Oid get_publication_oid(const char *pubname, bool missing_ok); extern char *get_publication_name(Oid pubid, bool missing_ok); extern Oid get_subscription_oid(const char *subname, bool missing_ok); extern char *get_subscription_name(Oid subid, bool missing_ok); +extern Oid get_subscription_database(Oid subid); #define type_is_array(typid) (get_element_type(typid) != InvalidOid) /* type_is_array_domain accepts both plain arrays and domains over arrays */ From a4663c08137449c09d65d033be8b297480bbb7ed Mon Sep 17 00:00:00 2001 From: reshke Date: Wed, 15 Apr 2026 16:17:44 +0500 Subject: [PATCH 44/45] Revert SSLKEYLOGFILE feature This reverts commit 2da74d8, a6c0bf9, parts of b7ab88d and 2970c75 Try to fix --- configure | 2 +- configure.ac | 2 +- doc/src/sgml/libpq.sgml | 24 --------- doc/src/sgml/release-18.sgml | 19 -------- meson.build | 1 - src/include/pg_config.h.in | 3 -- src/interfaces/libpq/fe-connect.c | 4 -- src/interfaces/libpq/fe-secure-openssl.c | 62 ------------------------ src/interfaces/libpq/libpq-int.h | 1 - src/test/ssl/t/001_ssltests.pl | 34 ------------- 10 files changed, 2 insertions(+), 150 deletions(-) diff --git a/configure b/configure index da9a74d77e8..0a055adb65c 100755 --- a/configure +++ b/configure @@ -13245,7 +13245,7 @@ fi done # Function introduced in OpenSSL 1.1.1, not in LibreSSL. - for ac_func in X509_get_signature_info SSL_CTX_set_num_tickets SSL_CTX_set_keylog_callback + for ac_func in X509_get_signature_info SSL_CTX_set_num_tickets do : as_ac_var=`$as_echo "ac_cv_func_$ac_func" | $as_tr_sh` ac_fn_c_check_func "$LINENO" "$ac_func" "$as_ac_var" diff --git a/configure.ac b/configure.ac index 6555179690b..4cc99ed59f1 100644 --- a/configure.ac +++ b/configure.ac @@ -1411,7 +1411,7 @@ if test "$with_ssl" = openssl ; then # Function introduced in OpenSSL 1.0.2, not in LibreSSL. AC_CHECK_FUNCS([SSL_CTX_set_cert_cb]) # Function introduced in OpenSSL 1.1.1, not in LibreSSL. - AC_CHECK_FUNCS([X509_get_signature_info SSL_CTX_set_num_tickets SSL_CTX_set_keylog_callback]) + AC_CHECK_FUNCS([X509_get_signature_info SSL_CTX_set_num_tickets]) AC_DEFINE([USE_OPENSSL], 1, [Define to 1 to build with OpenSSL support. (--with-ssl=openssl)]) elif test "$with_ssl" != no ; then AC_MSG_ERROR([--with-ssl must specify openssl]) diff --git a/doc/src/sgml/libpq.sgml b/doc/src/sgml/libpq.sgml index e50f0cbd8c7..ad74e65d69e 100644 --- a/doc/src/sgml/libpq.sgml +++ b/doc/src/sgml/libpq.sgml @@ -1918,30 +1918,6 @@ postgresql://%2Fvar%2Flib%2Fpostgresql/dbname - - sslkeylogfile - - - This parameter specifies the location where libpq - will log keys used in this SSL context. This is useful for debugging - PostgreSQL protocol interactions or client - connections using network inspection tools like - Wireshark. This parameter is ignored if an - SSL connection is not made, or if LibreSSL - is used (LibreSSL does not support key - logging). Keys are logged using the NSS - format. - - - Key logging will expose potentially sensitive information in the - keylog file. Keylog files should be handled with the same care as - files. - - - - - - sslpassword diff --git a/doc/src/sgml/release-18.sgml b/doc/src/sgml/release-18.sgml index ebb84c43f23..cf763142220 100644 --- a/doc/src/sgml/release-18.sgml +++ b/doc/src/sgml/release-18.sgml @@ -6029,25 +6029,6 @@ Author: Álvaro Herrera - - - - - Add libpq connection parameter sslkeylogfile - which dumps out SSL key material (Abhishek Chanda, - Daniel Gustafsson) - § - - - - This is useful for debugging. - - -