From 73f9d1d4d30ed392b761899b08d9b666afc76ca9 Mon Sep 17 00:00:00 2001 From: Nathan Bossart Date: Mon, 25 Nov 2024 16:36:37 -0600 Subject: [PATCH 01/54] pg_dump: Add dumpSchema and dumpData derivative flags. Various parts of pg_dump consult the --schema-only and --data-only options to determine whether to run a section of code. While this is simple enough for two mutually-exclusive options, it will become progressively more complicated as more options are added. In anticipation of that, this commit introduces new internal flags called dumpSchema and dumpData, which are derivatives of --schema-only and --data-only. This commit also removes the schemaOnly and dataOnly members from the dump/restore options structs to prevent their use elsewhere. Note that this change neither adds new user-facing command-line options nor changes the existing --schema-only and --data-only options. Author: Corey Huinker Reviewed-by: Jeff Davis Discussion: https://postgr.es/m/CADkLM%3DcQgghMJOS8EcAVBwRO4s1dUVtxGZv5gLPfZkQ1nL1gzA%40mail.gmail.com --- src/bin/pg_dump/pg_backup.h | 10 +- src/bin/pg_dump/pg_backup_archiver.c | 34 +++-- src/bin/pg_dump/pg_dump.c | 206 ++++++++++++++------------- src/bin/pg_dump/pg_restore.c | 14 +- 4 files changed, 141 insertions(+), 123 deletions(-) diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h index 609635ccbcb..113b4bf8b62 100644 --- a/src/bin/pg_dump/pg_backup.h +++ b/src/bin/pg_dump/pg_backup.h @@ -115,8 +115,6 @@ typedef struct _restoreOptions int strict_names; const char *filename; - int dataOnly; - int schemaOnly; int dumpSections; int verbose; int aclsSkip; @@ -159,6 +157,9 @@ typedef struct _restoreOptions int binary_upgrade; char *restrict_key; + /* flags derived from the user-settable flags */ + bool dumpSchema; + bool dumpData; } RestoreOptions; typedef struct _dumpOptions @@ -168,8 +169,6 @@ typedef struct _dumpOptions int binary_upgrade; /* various user-settable parameters */ - bool schemaOnly; - bool dataOnly; int dumpSections; /* bitmask of chosen sections */ bool aclsSkip; const char *lockWaitTimeout; @@ -207,6 +206,9 @@ typedef struct _dumpOptions int do_nothing; char *restrict_key; + /* flags derived from the user-settable flags */ + bool dumpSchema; + bool dumpData; } DumpOptions; /* diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c index 758ee098c2e..8485ff72b11 100644 --- a/src/bin/pg_dump/pg_backup_archiver.c +++ b/src/bin/pg_dump/pg_backup_archiver.c @@ -146,6 +146,8 @@ InitDumpOptions(DumpOptions *opts) opts->include_everything = true; opts->cparams.promptPassword = TRI_DEFAULT; opts->dumpSections = DUMP_UNSECTIONED; + opts->dumpSchema = true; + opts->dumpData = true; } /* @@ -164,8 +166,8 @@ dumpOptionsFromRestoreOptions(RestoreOptions *ropt) dopt->cparams.username = ropt->cparams.username ? pg_strdup(ropt->cparams.username) : NULL; dopt->cparams.promptPassword = ropt->cparams.promptPassword; dopt->outputClean = ropt->dropSchema; - dopt->dataOnly = ropt->dataOnly; - dopt->schemaOnly = ropt->schemaOnly; + dopt->dumpData = ropt->dumpData; + dopt->dumpSchema = ropt->dumpSchema; dopt->if_exists = ropt->if_exists; dopt->column_inserts = ropt->column_inserts; dopt->dumpSections = ropt->dumpSections; @@ -419,12 +421,12 @@ RestoreArchive(Archive *AHX) * Work out if we have an implied data-only restore. This can happen if * the dump was data only or if the user has used a toc list to exclude * all of the schema data. All we do is look for schema entries - if none - * are found then we set the dataOnly flag. + * are found then we unset the dumpSchema flag. * * We could scan for wanted TABLE entries, but that is not the same as - * dataOnly. At this stage, it seems unnecessary (6-Mar-2001). + * data-only. At this stage, it seems unnecessary (6-Mar-2001). */ - if (!ropt->dataOnly) + if (ropt->dumpSchema) { int impliedDataOnly = 1; @@ -438,7 +440,7 @@ RestoreArchive(Archive *AHX) } if (impliedDataOnly) { - ropt->dataOnly = impliedDataOnly; + ropt->dumpSchema = false; pg_log_info("implied data-only restore"); } } @@ -843,7 +845,7 @@ restore_toc_entry(ArchiveHandle *AH, TocEntry *te, bool is_parallel) /* Dump any relevant dump warnings to stderr */ if (!ropt->suppressDumpWarnings && strcmp(te->desc, "WARNING") == 0) { - if (!ropt->dataOnly && te->defn != NULL && strlen(te->defn) != 0) + if (ropt->dumpSchema && te->defn != NULL && strlen(te->defn) != 0) pg_log_warning("warning from original dump file: %s", te->defn); else if (te->copyStmt != NULL && strlen(te->copyStmt) != 0) pg_log_warning("warning from original dump file: %s", te->copyStmt); @@ -1099,6 +1101,8 @@ NewRestoreOptions(void) opts->dumpSections = DUMP_UNSECTIONED; opts->compression_spec.algorithm = PG_COMPRESSION_NONE; opts->compression_spec.level = 0; + opts->dumpSchema = true; + opts->dumpData = true; return opts; } @@ -1109,7 +1113,7 @@ _disableTriggersIfNecessary(ArchiveHandle *AH, TocEntry *te) RestoreOptions *ropt = AH->public.ropt; /* This hack is only needed in a data-only restore */ - if (!ropt->dataOnly || !ropt->disable_triggers) + if (ropt->dumpSchema || !ropt->disable_triggers) return; pg_log_info("disabling triggers for %s", te->tag); @@ -1135,7 +1139,7 @@ _enableTriggersIfNecessary(ArchiveHandle *AH, TocEntry *te) RestoreOptions *ropt = AH->public.ropt; /* This hack is only needed in a data-only restore */ - if (!ropt->dataOnly || !ropt->disable_triggers) + if (ropt->dumpSchema || !ropt->disable_triggers) return; pg_log_info("enabling triggers for %s", te->tag); @@ -3196,13 +3200,13 @@ _tocEntryRequired(TocEntry *te, teSection curSection, ArchiveHandle *AH) if ((strcmp(te->desc, "") == 0) && (strcmp(te->tag, "Max OID") == 0)) return 0; - /* Mask it if we only want schema */ - if (ropt->schemaOnly) + /* Mask it if we don't want data */ + if (!ropt->dumpData) { /* - * The sequence_data option overrides schemaOnly for SEQUENCE SET. + * The sequence_data option overrides dumpData for SEQUENCE SET. * - * In binary-upgrade mode, even with schemaOnly set, we do not mask + * In binary-upgrade mode, even with dumpData unset, we do not mask * out large objects. (Only large object definitions, comments and * other metadata should be generated in binary-upgrade mode, not the * actual data, but that need not concern us here.) @@ -3220,8 +3224,8 @@ _tocEntryRequired(TocEntry *te, teSection curSection, ArchiveHandle *AH) res = res & REQ_SCHEMA; } - /* Mask it if we only want data */ - if (ropt->dataOnly) + /* Mask it if we don't want schema */ + if (!ropt->dumpSchema) res = res & REQ_DATA; return res; diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index e59d56262c8..a0c733c477d 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -373,6 +373,8 @@ main(int argc, char **argv) char *error_detail = NULL; bool user_compression_defined = false; DataDirSyncMethod sync_method = DATA_DIR_SYNC_METHOD_FSYNC; + bool data_only = false; + bool schema_only = false; static DumpOptions dopt; @@ -489,7 +491,7 @@ main(int argc, char **argv) switch (c) { case 'a': /* Dump data only */ - dopt.dataOnly = true; + data_only = true; break; case 'b': /* Dump LOs */ @@ -562,7 +564,7 @@ main(int argc, char **argv) break; case 's': /* dump schema only */ - dopt.schemaOnly = true; + schema_only = true; break; case 'S': /* Username for superuser in plain text output */ @@ -730,21 +732,25 @@ main(int argc, char **argv) if (dopt.binary_upgrade) dopt.sequence_data = 1; - if (dopt.dataOnly && dopt.schemaOnly) + if (data_only && schema_only) pg_fatal("options -s/--schema-only and -a/--data-only cannot be used together"); - if (dopt.schemaOnly && foreign_servers_include_patterns.head != NULL) + if (schema_only && foreign_servers_include_patterns.head != NULL) pg_fatal("options -s/--schema-only and --include-foreign-data cannot be used together"); if (numWorkers > 1 && foreign_servers_include_patterns.head != NULL) pg_fatal("option --include-foreign-data is not supported with parallel backup"); - if (dopt.dataOnly && dopt.outputClean) + if (data_only && dopt.outputClean) pg_fatal("options -c/--clean and -a/--data-only cannot be used together"); if (dopt.if_exists && !dopt.outputClean) pg_fatal("option --if-exists requires option -c/--clean"); + /* set derivative flags */ + dopt.dumpSchema = (!data_only); + dopt.dumpData = (!schema_only); + /* * --inserts are already implied above if --column-inserts or * --rows-per-insert were specified. @@ -941,7 +947,7 @@ main(int argc, char **argv) * -s means "schema only" and LOs are data, not schema, so we never * include LOs when -s is used. */ - if (dopt.include_everything && !dopt.schemaOnly && !dopt.dontOutputLOs) + if (dopt.include_everything && dopt.dumpData && !dopt.dontOutputLOs) dopt.outputLOs = true; /* @@ -955,15 +961,15 @@ main(int argc, char **argv) */ tblinfo = getSchemaData(fout, &numTables); - if (!dopt.schemaOnly) + if (dopt.dumpData) { getTableData(&dopt, tblinfo, numTables, 0); buildMatViewRefreshDependencies(fout); - if (dopt.dataOnly) + if (!dopt.dumpSchema) getTableDataFKConstraints(); } - if (dopt.schemaOnly && dopt.sequence_data) + if (!dopt.dumpData && dopt.sequence_data) getTableData(&dopt, tblinfo, numTables, RELKIND_SEQUENCE); /* @@ -1048,8 +1054,8 @@ main(int argc, char **argv) ropt->cparams.username = dopt.cparams.username ? pg_strdup(dopt.cparams.username) : NULL; ropt->cparams.promptPassword = dopt.cparams.promptPassword; ropt->dropSchema = dopt.outputClean; - ropt->dataOnly = dopt.dataOnly; - ropt->schemaOnly = dopt.schemaOnly; + ropt->dumpData = dopt.dumpData; + ropt->dumpSchema = dopt.dumpSchema; ropt->if_exists = dopt.if_exists; ropt->column_inserts = dopt.column_inserts; ropt->dumpSections = dopt.dumpSections; @@ -1947,7 +1953,7 @@ selectDumpableType(TypeInfo *tyinfo, Archive *fout) * Mark a default ACL as to be dumped or not * * For per-schema default ACLs, dump if the schema is to be dumped. - * Otherwise dump if we are dumping "everything". Note that dataOnly + * Otherwise dump if we are dumping "everything". Note that dumpSchema * and aclsSkip are checked separately. */ static void @@ -4125,8 +4131,8 @@ dumpPolicy(Archive *fout, const PolicyInfo *polinfo) const char *cmd; char *tag; - /* Do nothing in data-only dump */ - if (dopt->dataOnly) + /* Do nothing if not dumping schema */ + if (!dopt->dumpSchema) return; /* @@ -4344,8 +4350,8 @@ dumpPublication(Archive *fout, const PublicationInfo *pubinfo) char *qpubname; bool first = true; - /* Do nothing in data-only dump */ - if (dopt->dataOnly) + /* Do nothing if not dumping schema */ + if (!dopt->dumpSchema) return; delq = createPQExpBuffer(); @@ -4659,8 +4665,8 @@ dumpPublicationNamespace(Archive *fout, const PublicationSchemaInfo *pubsinfo) PQExpBuffer query; char *tag; - /* Do nothing in data-only dump */ - if (dopt->dataOnly) + /* Do nothing if not dumping schema */ + if (!dopt->dumpSchema) return; tag = psprintf("%s %s", pubinfo->dobj.name, schemainfo->dobj.name); @@ -4702,8 +4708,8 @@ dumpPublicationTable(Archive *fout, const PublicationRelInfo *pubrinfo) PQExpBuffer query; char *tag; - /* Do nothing in data-only dump */ - if (dopt->dataOnly) + /* Do nothing if not dumping schema */ + if (!dopt->dumpSchema) return; tag = psprintf("%s %s", pubinfo->dobj.name, tbinfo->dobj.name); @@ -5088,8 +5094,8 @@ dumpSubscriptionTable(Archive *fout, const SubRelInfo *subrinfo) PQExpBuffer query; char *tag; - /* Do nothing in data-only dump */ - if (dopt->dataOnly) + /* Do nothing if not dumping schema */ + if (!dopt->dumpSchema) return; Assert(fout->dopt->binary_upgrade && fout->remoteVersion >= 170000); @@ -5162,8 +5168,8 @@ dumpSubscription(Archive *fout, const SubscriptionInfo *subinfo) int i; char two_phase_disabled[] = {LOGICALREP_TWOPHASE_STATE_DISABLED, '\0'}; - /* Do nothing in data-only dump */ - if (dopt->dataOnly) + /* Do nothing if not dumping schema */ + if (!dopt->dumpSchema) return; delq = createPQExpBuffer(); @@ -7379,8 +7385,8 @@ getPartitioningInfo(Archive *fout) /* hash partitioning didn't exist before v11 */ if (fout->remoteVersion < 110000) return; - /* needn't bother if schema-only dump */ - if (fout->dopt->schemaOnly) + /* needn't bother if not dumping data */ + if (!fout->dopt->dumpData) return; query = createPQExpBuffer(); @@ -9078,7 +9084,7 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables) * Now get info about column defaults. This is skipped for a data-only * dump, as it is only needed for table schemas. */ - if (!dopt->dataOnly && tbloids->len > 1) + if (dopt->dumpSchema && tbloids->len > 1) { AttrDefInfo *attrdefs; int numDefaults; @@ -9208,7 +9214,7 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables) * Get info about table CHECK constraints. This is skipped for a * data-only dump, as it is only needed for table schemas. */ - if (!dopt->dataOnly && checkoids->len > 2) + if (dopt->dumpSchema && checkoids->len > 2) { ConstraintInfo *constrs; int numConstrs; @@ -10160,13 +10166,13 @@ dumpCommentExtended(Archive *fout, const char *type, /* Comments are schema not data ... except LO comments are data */ if (strcmp(type, "LARGE OBJECT") != 0) { - if (dopt->dataOnly) + if (!dopt->dumpSchema) return; } else { /* We do dump LO comments in binary-upgrade mode */ - if (dopt->schemaOnly && !dopt->binary_upgrade) + if (!dopt->dumpData && !dopt->binary_upgrade) return; } @@ -10273,7 +10279,7 @@ dumpTableComment(Archive *fout, const TableInfo *tbinfo, return; /* Comments are SCHEMA not data */ - if (dopt->dataOnly) + if (!dopt->dumpSchema) return; /* Search for comments associated with relation, using table */ @@ -10719,8 +10725,8 @@ dumpNamespace(Archive *fout, const NamespaceInfo *nspinfo) PQExpBuffer delq; char *qnspname; - /* Do nothing in data-only dump */ - if (dopt->dataOnly) + /* Do nothing if not dumping schema */ + if (!dopt->dumpSchema) return; q = createPQExpBuffer(); @@ -10796,8 +10802,8 @@ dumpExtension(Archive *fout, const ExtensionInfo *extinfo) PQExpBuffer delq; char *qextname; - /* Do nothing in data-only dump */ - if (dopt->dataOnly) + /* Do nothing if not dumping schema */ + if (!dopt->dumpSchema) return; q = createPQExpBuffer(); @@ -10921,8 +10927,8 @@ dumpType(Archive *fout, const TypeInfo *tyinfo) { DumpOptions *dopt = fout->dopt; - /* Do nothing in data-only dump */ - if (dopt->dataOnly) + /* Do nothing if not dumping schema */ + if (!dopt->dumpSchema) return; /* Dump out in proper style */ @@ -12084,8 +12090,8 @@ dumpShellType(Archive *fout, const ShellTypeInfo *stinfo) DumpOptions *dopt = fout->dopt; PQExpBuffer q; - /* Do nothing in data-only dump */ - if (dopt->dataOnly) + /* Do nothing if not dumping schema */ + if (!dopt->dumpSchema) return; q = createPQExpBuffer(); @@ -12136,8 +12142,8 @@ dumpProcLang(Archive *fout, const ProcLangInfo *plang) FuncInfo *inlineInfo = NULL; FuncInfo *validatorInfo = NULL; - /* Do nothing in data-only dump */ - if (dopt->dataOnly) + /* Do nothing if not dumping schema */ + if (!dopt->dumpSchema) return; /* @@ -12344,8 +12350,8 @@ dumpFunc(Archive *fout, const FuncInfo *finfo) int nconfigitems = 0; const char *keyword; - /* Do nothing in data-only dump */ - if (dopt->dataOnly) + /* Do nothing if not dumping schema */ + if (!dopt->dumpSchema) return; query = createPQExpBuffer(); @@ -12736,8 +12742,8 @@ dumpCast(Archive *fout, const CastInfo *cast) const char *sourceType; const char *targetType; - /* Do nothing in data-only dump */ - if (dopt->dataOnly) + /* Do nothing if not dumping schema */ + if (!dopt->dumpSchema) return; /* Cannot dump if we don't have the cast function's info */ @@ -12842,8 +12848,8 @@ dumpTransform(Archive *fout, const TransformInfo *transform) char *lanname; const char *transformType; - /* Do nothing in data-only dump */ - if (dopt->dataOnly) + /* Do nothing if not dumping schema */ + if (!dopt->dumpSchema) return; /* Cannot dump if we don't have the transform functions' info */ @@ -12991,8 +12997,8 @@ dumpOpr(Archive *fout, const OprInfo *oprinfo) char *oprregproc; char *oprref; - /* Do nothing in data-only dump */ - if (dopt->dataOnly) + /* Do nothing if not dumping schema */ + if (!dopt->dumpSchema) return; /* @@ -13278,8 +13284,8 @@ dumpAccessMethod(Archive *fout, const AccessMethodInfo *aminfo) PQExpBuffer delq; char *qamname; - /* Do nothing in data-only dump */ - if (dopt->dataOnly) + /* Do nothing if not dumping schema */ + if (!dopt->dumpSchema) return; q = createPQExpBuffer(); @@ -13381,8 +13387,8 @@ dumpOpclass(Archive *fout, const OpclassInfo *opcinfo) bool needComma; int i; - /* Do nothing in data-only dump */ - if (dopt->dataOnly) + /* Do nothing if not dumping schema */ + if (!dopt->dumpSchema) return; query = createPQExpBuffer(); @@ -13652,8 +13658,8 @@ dumpOpfamily(Archive *fout, const OpfamilyInfo *opfinfo) bool needComma; int i; - /* Do nothing in data-only dump */ - if (dopt->dataOnly) + /* Do nothing if not dumping schema */ + if (!dopt->dumpSchema) return; query = createPQExpBuffer(); @@ -13859,8 +13865,8 @@ dumpCollation(Archive *fout, const CollInfo *collinfo) const char *colllocale; const char *collicurules; - /* Do nothing in data-only dump */ - if (dopt->dataOnly) + /* Do nothing if not dumping schema */ + if (!dopt->dumpSchema) return; query = createPQExpBuffer(); @@ -14113,8 +14119,8 @@ dumpConversion(Archive *fout, const ConvInfo *convinfo) const char *conproc; bool condefault; - /* Do nothing in data-only dump */ - if (dopt->dataOnly) + /* Do nothing if not dumping schema */ + if (!dopt->dumpSchema) return; query = createPQExpBuffer(); @@ -14261,8 +14267,8 @@ dumpAgg(Archive *fout, const AggInfo *agginfo) const char *proparallel; char defaultfinalmodify; - /* Do nothing in data-only dump */ - if (dopt->dataOnly) + /* Do nothing if not dumping schema */ + if (!dopt->dumpSchema) return; query = createPQExpBuffer(); @@ -14591,8 +14597,8 @@ dumpTSParser(Archive *fout, const TSParserInfo *prsinfo) PQExpBuffer delq; char *qprsname; - /* Do nothing in data-only dump */ - if (dopt->dataOnly) + /* Do nothing if not dumping schema */ + if (!dopt->dumpSchema) return; q = createPQExpBuffer(); @@ -14659,8 +14665,8 @@ dumpTSDictionary(Archive *fout, const TSDictInfo *dictinfo) char *nspname; char *tmplname; - /* Do nothing in data-only dump */ - if (dopt->dataOnly) + /* Do nothing if not dumping schema */ + if (!dopt->dumpSchema) return; q = createPQExpBuffer(); @@ -14735,8 +14741,8 @@ dumpTSTemplate(Archive *fout, const TSTemplateInfo *tmplinfo) PQExpBuffer delq; char *qtmplname; - /* Do nothing in data-only dump */ - if (dopt->dataOnly) + /* Do nothing if not dumping schema */ + if (!dopt->dumpSchema) return; q = createPQExpBuffer(); @@ -14801,8 +14807,8 @@ dumpTSConfig(Archive *fout, const TSConfigInfo *cfginfo) int i_tokenname; int i_dictname; - /* Do nothing in data-only dump */ - if (dopt->dataOnly) + /* Do nothing if not dumping schema */ + if (!dopt->dumpSchema) return; q = createPQExpBuffer(); @@ -14913,8 +14919,8 @@ dumpForeignDataWrapper(Archive *fout, const FdwInfo *fdwinfo) PQExpBuffer delq; char *qfdwname; - /* Do nothing in data-only dump */ - if (dopt->dataOnly) + /* Do nothing if not dumping schema */ + if (!dopt->dumpSchema) return; q = createPQExpBuffer(); @@ -14986,8 +14992,8 @@ dumpForeignServer(Archive *fout, const ForeignServerInfo *srvinfo) char *qsrvname; char *fdwname; - /* Do nothing in data-only dump */ - if (dopt->dataOnly) + /* Do nothing if not dumping schema */ + if (!dopt->dumpSchema) return; q = createPQExpBuffer(); @@ -15177,8 +15183,8 @@ dumpDefaultACL(Archive *fout, const DefaultACLInfo *daclinfo) PQExpBuffer tag; const char *type; - /* Do nothing in data-only dump, or if we're skipping ACLs */ - if (dopt->dataOnly || dopt->aclsSkip) + /* Do nothing if not dumping schema, or if we're skipping ACLs */ + if (!dopt->dumpSchema || dopt->aclsSkip) return; q = createPQExpBuffer(); @@ -15278,7 +15284,7 @@ dumpACL(Archive *fout, DumpId objDumpId, DumpId altDumpId, return InvalidDumpId; /* --data-only skips ACLs *except* large object ACLs */ - if (dopt->dataOnly && strcmp(type, "LARGE OBJECT") != 0) + if (!dopt->dumpSchema && strcmp(type, "LARGE OBJECT") != 0) return InvalidDumpId; sql = createPQExpBuffer(); @@ -15407,13 +15413,13 @@ dumpSecLabel(Archive *fout, const char *type, const char *name, */ if (strcmp(type, "LARGE OBJECT") != 0) { - if (dopt->dataOnly) + if (!dopt->dumpSchema) return; } else { /* We do dump large object security labels in binary-upgrade mode */ - if (dopt->schemaOnly && !dopt->binary_upgrade) + if (!dopt->dumpData && !dopt->binary_upgrade) return; } @@ -15481,7 +15487,7 @@ dumpTableSecLabel(Archive *fout, const TableInfo *tbinfo, const char *reltypenam return; /* SecLabel are SCHEMA not data */ - if (dopt->dataOnly) + if (!dopt->dumpSchema) return; /* Search for comments associated with relation, using table */ @@ -15720,8 +15726,8 @@ dumpTable(Archive *fout, const TableInfo *tbinfo) DumpId tableAclDumpId = InvalidDumpId; char *namecopy; - /* Do nothing in data-only dump */ - if (dopt->dataOnly) + /* Do nothing if not dumping schema */ + if (!dopt->dumpSchema) return; if (tbinfo->dobj.dump & DUMP_COMPONENT_DEFINITION) @@ -16812,8 +16818,8 @@ dumpTableAttach(Archive *fout, const TableAttachInfo *attachinfo) PGresult *res; char *partbound; - /* Do nothing in data-only dump */ - if (dopt->dataOnly) + /* Do nothing if not dumping schema */ + if (!dopt->dumpSchema) return; q = createPQExpBuffer(); @@ -16884,8 +16890,8 @@ dumpAttrDef(Archive *fout, const AttrDefInfo *adinfo) char *tag; char *foreign; - /* Do nothing in data-only dump */ - if (dopt->dataOnly) + /* Do nothing if not dumping schema */ + if (!dopt->dumpSchema) return; /* Skip if not "separate"; it was dumped in the table's definition */ @@ -16973,8 +16979,8 @@ dumpIndex(Archive *fout, const IndxInfo *indxinfo) char *qindxname; char *qqindxname; - /* Do nothing in data-only dump */ - if (dopt->dataOnly) + /* Do nothing if not dumping schema */ + if (!dopt->dumpSchema) return; q = createPQExpBuffer(); @@ -17116,8 +17122,8 @@ dumpIndex(Archive *fout, const IndxInfo *indxinfo) static void dumpIndexAttach(Archive *fout, const IndexAttachInfo *attachinfo) { - /* Do nothing in data-only dump */ - if (fout->dopt->dataOnly) + /* Do nothing if not dumping schema */ + if (!fout->dopt->dumpSchema) return; if (attachinfo->partitionIdx->dobj.dump & DUMP_COMPONENT_DEFINITION) @@ -17167,8 +17173,8 @@ dumpStatisticsExt(Archive *fout, const StatsExtInfo *statsextinfo) PGresult *res; char *stxdef; - /* Do nothing in data-only dump */ - if (dopt->dataOnly) + /* Do nothing if not dumping schema */ + if (!dopt->dumpSchema) return; q = createPQExpBuffer(); @@ -17243,8 +17249,8 @@ dumpConstraint(Archive *fout, const ConstraintInfo *coninfo) char *tag = NULL; char *foreign; - /* Do nothing in data-only dump */ - if (dopt->dataOnly) + /* Do nothing if not dumping schema */ + if (!dopt->dumpSchema) return; q = createPQExpBuffer(); @@ -17901,8 +17907,8 @@ dumpTrigger(Archive *fout, const TriggerInfo *tginfo) char *qtabname; char *tag; - /* Do nothing in data-only dump */ - if (dopt->dataOnly) + /* Do nothing if not dumping schema */ + if (!dopt->dumpSchema) return; query = createPQExpBuffer(); @@ -18023,8 +18029,8 @@ dumpEventTrigger(Archive *fout, const EventTriggerInfo *evtinfo) PQExpBuffer delqry; char *qevtname; - /* Do nothing in data-only dump */ - if (dopt->dataOnly) + /* Do nothing if not dumping schema */ + if (!dopt->dumpSchema) return; query = createPQExpBuffer(); @@ -18119,8 +18125,8 @@ dumpRule(Archive *fout, const RuleInfo *rinfo) PGresult *res; char *tag; - /* Do nothing in data-only dump */ - if (dopt->dataOnly) + /* Do nothing if not dumping schema */ + if (!dopt->dumpSchema) return; /* @@ -18386,7 +18392,7 @@ processExtensionTables(Archive *fout, ExtensionInfo extinfo[], * objects for them, ensuring their data will be dumped even though the * tables themselves won't be. * - * Note that we create TableDataInfo objects even in schemaOnly mode, ie, + * Note that we create TableDataInfo objects even in schema-only mode, ie, * user data in a configuration table is treated like schema data. This * seems appropriate since system data in a config table would get * reloaded by CREATE EXTENSION. If the extension is not listed in the diff --git a/src/bin/pg_dump/pg_restore.c b/src/bin/pg_dump/pg_restore.c index fc6fa923231..ae9d20cfe82 100644 --- a/src/bin/pg_dump/pg_restore.c +++ b/src/bin/pg_dump/pg_restore.c @@ -76,6 +76,8 @@ main(int argc, char **argv) static int no_security_labels = 0; static int no_subscriptions = 0; static int strict_names = 0; + bool data_only = false; + bool schema_only = false; struct option cmdopts[] = { {"clean", 0, NULL, 'c'}, @@ -162,7 +164,7 @@ main(int argc, char **argv) switch (c) { case 'a': /* Dump data only */ - opts->dataOnly = 1; + data_only = true; break; case 'c': /* clean (i.e., drop) schema prior to create */ opts->dropSchema = 1; @@ -238,7 +240,7 @@ main(int argc, char **argv) simple_string_list_append(&opts->triggerNames, optarg); break; case 's': /* dump schema only */ - opts->schemaOnly = 1; + schema_only = true; break; case 'S': /* Superuser username */ if (strlen(optarg) != 0) @@ -361,10 +363,10 @@ main(int argc, char **argv) pg_fatal("invalid restrict key"); } - if (opts->dataOnly && opts->schemaOnly) + if (data_only && schema_only) pg_fatal("options -s/--schema-only and -a/--data-only cannot be used together"); - if (opts->dataOnly && opts->dropSchema) + if (data_only && opts->dropSchema) pg_fatal("options -c/--clean and -a/--data-only cannot be used together"); if (opts->single_txn && opts->txn_size > 0) @@ -381,6 +383,10 @@ main(int argc, char **argv) if (opts->single_txn && numWorkers > 1) pg_fatal("cannot specify both --single-transaction and multiple jobs"); + /* set derivative flags */ + opts->dumpSchema = (!data_only); + opts->dumpData = (!schema_only); + opts->disable_triggers = disable_triggers; opts->enable_row_security = enable_row_security; opts->noDataForFailedTables = no_data_for_failed_tables; From 152b4a9cd065b3c7333faa87f6fbe4f65ea13ac2 Mon Sep 17 00:00:00 2001 From: Nathan Bossart Date: Mon, 1 Jul 2024 10:18:26 -0500 Subject: [PATCH 02/54] Add --no-sync to pg_upgrade's uses of pg_dump and pg_dumpall. There's no reason to ensure that the files pg_upgrade generates with pg_dump and pg_dumpall have been written safely to disk. If there is a crash during pg_upgrade, the upgrade must be restarted from the beginning; dump files left behind by previous pg_upgrade attempts cannot be reused. Reviewed-by: Peter Eisentraut, Tom Lane, Michael Paquier, Daniel Gustafsson Discussion: https://postgr.es/m/20240503171348.GA1731524%40nathanxps13 --- src/bin/pg_upgrade/dump.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/bin/pg_upgrade/dump.c b/src/bin/pg_upgrade/dump.c index b14c4ce0ee2..477436290e8 100644 --- a/src/bin/pg_upgrade/dump.c +++ b/src/bin/pg_upgrade/dump.c @@ -22,7 +22,7 @@ generate_old_dump(void) /* run new pg_dumpall binary for globals */ exec_prog(UTILITY_LOG_FILE, NULL, true, true, "\"%s/pg_dumpall\" %s%s --globals-only --quote-all-identifiers " - "--binary-upgrade %s -f \"%s/%s\"", + "--binary-upgrade %s --no-sync -f \"%s/%s\"", new_cluster.bindir, cluster_conn_opts(&old_cluster), protocol_negotiation_supported(&old_cluster) ? "" : " -d \"max_protocol_version=3.0\"", log_opts.verbose ? "--verbose" : "", @@ -57,7 +57,7 @@ generate_old_dump(void) parallel_exec_prog(log_file_name, NULL, "\"%s/pg_dump\" %s --schema-only --quote-all-identifiers " - "--binary-upgrade --format=custom %s --file=\"%s/%s\" %s", + "--binary-upgrade --format=custom %s --no-sync --file=\"%s/%s\" %s", new_cluster.bindir, cluster_conn_opts(&old_cluster), log_opts.verbose ? "--verbose" : "", log_opts.dumpdir, From 1e20ec93c77b4c393024d24c8f72219b3344340e Mon Sep 17 00:00:00 2001 From: Jeff Davis Date: Fri, 11 Oct 2024 16:55:11 -0700 Subject: [PATCH 03/54] Create functions pg_set_relation_stats, pg_clear_relation_stats. These functions are used to tweak statistics on any relation, provided that the user has MAINTAIN privilege on the relation, or is the database owner. Bump catalog version. Author: Corey Huinker Discussion: https://postgr.es/m/CADkLM=eErgzn7ECDpwFcptJKOk9SxZEk5Pot4d94eVTZsvj3gw@mail.gmail.com --- doc/src/sgml/func.sgml | 94 +++++++++ src/backend/catalog/system_functions.sql | 10 + src/backend/statistics/Makefile | 4 +- src/backend/statistics/meson.build | 2 + src/backend/statistics/relation_stats.c | 210 +++++++++++++++++++++ src/backend/statistics/stat_utils.c | 94 +++++++++ src/include/catalog/pg_proc.dat | 16 +- src/include/statistics/stat_utils.h | 29 +++ src/test/regress/expected/stats_import.out | 143 ++++++++++++++ src/test/regress/parallel_schedule | 2 +- src/test/regress/sql/stats_import.sql | 98 ++++++++++ 11 files changed, 699 insertions(+), 3 deletions(-) create mode 100644 src/backend/statistics/relation_stats.c create mode 100644 src/backend/statistics/stat_utils.c create mode 100644 src/include/statistics/stat_utils.h create mode 100644 src/test/regress/expected/stats_import.out create mode 100644 src/test/regress/sql/stats_import.sql diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 3dff7be675f..3a83b989f83 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -29950,6 +29950,100 @@ postgres=# SELECT '0/0'::pg_lsn + pd.segment_number * ps.setting::int + :offset + + lists functions used to + manipulate statistics. + + + Changes made by these statistics manipulation functions are likely to be + overwritten by autovacuum (or manual + VACUUM or ANALYZE) and should be + considered temporary. + + + + + + Database Object Statistics Manipulation Functions + + + + + Function + + + Description + + + + + + + + + + pg_set_relation_stats + + pg_set_relation_stats ( + relation regclass + , relpages integer + , reltuples real + , relallvisible integer ) + boolean + + + Updates relation-level statistics for the given relation to the + specified values. The parameters correspond to columns in pg_class. Unspecified + or NULL values leave the setting + unchanged. Returns true if a change was made; + false otherwise. + + + Ordinarily, these statistics are collected automatically or updated + as a part of or , so it's not necessary to call this + function. However, it may be useful when testing the effects of + statistics on the planner to understand or anticipate plan changes. + + + The caller must have the MAINTAIN privilege on + the table or be the owner of the database. + + + The value of relpages must be greater than + or equal to 0, + reltuples must be greater than or equal to + -1.0, and relallvisible + must be greater than or equal to 0. + + + + + + + + + pg_clear_relation_stats + + pg_clear_relation_stats ( relation regclass ) + boolean + + + Clears table-level statistics for the given relation, as though the + table was newly created. Returns true if a change + was made; false otherwise. + + + The caller must have the MAINTAIN privilege on + the table or be the owner of the database. + + + + + +
+ lists functions that provide information about the structure of partitioned tables. diff --git a/src/backend/catalog/system_functions.sql b/src/backend/catalog/system_functions.sql index ae099e328c2..0b13eec0ca8 100644 --- a/src/backend/catalog/system_functions.sql +++ b/src/backend/catalog/system_functions.sql @@ -636,6 +636,16 @@ LANGUAGE INTERNAL CALLED ON NULL INPUT VOLATILE PARALLEL SAFE AS 'pg_stat_reset_slru'; +CREATE OR REPLACE FUNCTION + pg_set_relation_stats(relation regclass, + relpages integer DEFAULT NULL, + reltuples real DEFAULT NULL, + relallvisible integer DEFAULT NULL) +RETURNS bool +LANGUAGE INTERNAL +CALLED ON NULL INPUT VOLATILE +AS 'pg_set_relation_stats'; + -- -- The default permissions for functions mean that anyone can execute them. -- A number of functions shouldn't be executable by just anyone, but rather diff --git a/src/backend/statistics/Makefile b/src/backend/statistics/Makefile index 89cf8c27973..041f5f8a581 100644 --- a/src/backend/statistics/Makefile +++ b/src/backend/statistics/Makefile @@ -16,6 +16,8 @@ OBJS = \ dependencies.o \ extended_stats.o \ mcv.o \ - mvdistinct.o + mvdistinct.o \ + relation_stats.o \ + stat_utils.o include $(top_srcdir)/src/backend/common.mk diff --git a/src/backend/statistics/meson.build b/src/backend/statistics/meson.build index 73b29a3d50a..23648b3775f 100644 --- a/src/backend/statistics/meson.build +++ b/src/backend/statistics/meson.build @@ -5,4 +5,6 @@ backend_sources += files( 'extended_stats.c', 'mcv.c', 'mvdistinct.c', + 'relation_stats.c', + 'stat_utils.c' ) diff --git a/src/backend/statistics/relation_stats.c b/src/backend/statistics/relation_stats.c new file mode 100644 index 00000000000..ffa3d83a878 --- /dev/null +++ b/src/backend/statistics/relation_stats.c @@ -0,0 +1,210 @@ +/*------------------------------------------------------------------------- + * relation_stats.c + * + * PostgreSQL relation statistics manipulation + * + * Code supporting the direct import of relation statistics, similar to + * what is done by the ANALYZE command. + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/statistics/relation_stats.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "access/heapam.h" +#include "catalog/indexing.h" +#include "statistics/stat_utils.h" +#include "utils/fmgrprotos.h" +#include "utils/syscache.h" + +#define DEFAULT_RELPAGES Int32GetDatum(0) +#define DEFAULT_RELTUPLES Float4GetDatum(-1.0) +#define DEFAULT_RELALLVISIBLE Int32GetDatum(0) + +/* + * Positional argument numbers, names, and types for + * relation_statistics_update(). + */ + +enum relation_stats_argnum +{ + RELATION_ARG = 0, + RELPAGES_ARG, + RELTUPLES_ARG, + RELALLVISIBLE_ARG, + NUM_RELATION_STATS_ARGS +}; + +static struct StatsArgInfo relarginfo[] = +{ + [RELATION_ARG] = {"relation", REGCLASSOID}, + [RELPAGES_ARG] = {"relpages", INT4OID}, + [RELTUPLES_ARG] = {"reltuples", FLOAT4OID}, + [RELALLVISIBLE_ARG] = {"relallvisible", INT4OID}, + [NUM_RELATION_STATS_ARGS] = {0} +}; + +static bool relation_statistics_update(FunctionCallInfo fcinfo, int elevel); + +/* + * Internal function for modifying statistics for a relation. + */ +static bool +relation_statistics_update(FunctionCallInfo fcinfo, int elevel) +{ + Oid reloid; + Relation crel; + HeapTuple ctup; + Form_pg_class pgcform; + int replaces[3] = {0}; + Datum values[3] = {0}; + bool nulls[3] = {0}; + int ncols = 0; + TupleDesc tupdesc; + HeapTuple newtup; + + + stats_check_required_arg(fcinfo, relarginfo, RELATION_ARG); + reloid = PG_GETARG_OID(RELATION_ARG); + + stats_lock_check_privileges(reloid); + + /* + * Take RowExclusiveLock on pg_class, consistent with + * vac_update_relstats(). + */ + crel = table_open(RelationRelationId, RowExclusiveLock); + + tupdesc = RelationGetDescr(crel); + ctup = SearchSysCacheCopy1(RELOID, ObjectIdGetDatum(reloid)); + if (!HeapTupleIsValid(ctup)) + { + ereport(elevel, + (errcode(ERRCODE_OBJECT_IN_USE), + errmsg("pg_class entry for relid %u not found", reloid))); + table_close(crel, RowExclusiveLock); + return false; + } + + pgcform = (Form_pg_class) GETSTRUCT(ctup); + + /* relpages */ + if (!PG_ARGISNULL(RELPAGES_ARG)) + { + int32 relpages = PG_GETARG_INT32(RELPAGES_ARG); + + if (relpages < -1) + { + ereport(elevel, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("relpages cannot be < -1"))); + table_close(crel, RowExclusiveLock); + return false; + } + + if (relpages != pgcform->relpages) + { + replaces[ncols] = Anum_pg_class_relpages; + values[ncols] = Int32GetDatum(relpages); + ncols++; + } + } + + if (!PG_ARGISNULL(RELTUPLES_ARG)) + { + float reltuples = PG_GETARG_FLOAT4(RELTUPLES_ARG); + + if (reltuples < -1.0) + { + ereport(elevel, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("reltuples cannot be < -1.0"))); + table_close(crel, RowExclusiveLock); + return false; + } + + if (reltuples != pgcform->reltuples) + { + replaces[ncols] = Anum_pg_class_reltuples; + values[ncols] = Float4GetDatum(reltuples); + ncols++; + } + } + + if (!PG_ARGISNULL(RELALLVISIBLE_ARG)) + { + int32 relallvisible = PG_GETARG_INT32(RELALLVISIBLE_ARG); + + if (relallvisible < 0) + { + ereport(elevel, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("relallvisible cannot be < 0"))); + table_close(crel, RowExclusiveLock); + return false; + } + + if (relallvisible != pgcform->relallvisible) + { + replaces[ncols] = Anum_pg_class_relallvisible; + values[ncols] = Int32GetDatum(relallvisible); + ncols++; + } + } + + /* only update pg_class if there is a meaningful change */ + if (ncols == 0) + { + table_close(crel, RowExclusiveLock); + return false; + } + + newtup = heap_modify_tuple_by_cols(ctup, tupdesc, ncols, replaces, values, + nulls); + + CatalogTupleUpdate(crel, &newtup->t_self, newtup); + heap_freetuple(newtup); + + /* release the lock, consistent with vac_update_relstats() */ + table_close(crel, RowExclusiveLock); + + return true; +} + +/* + * Set statistics for a given pg_class entry. + */ +Datum +pg_set_relation_stats(PG_FUNCTION_ARGS) +{ + PG_RETURN_BOOL(relation_statistics_update(fcinfo, ERROR)); +} + +/* + * Clear statistics for a given pg_class entry; that is, set back to initial + * stats for a newly-created table. + */ +Datum +pg_clear_relation_stats(PG_FUNCTION_ARGS) +{ + LOCAL_FCINFO(newfcinfo, 4); + + InitFunctionCallInfoData(*newfcinfo, NULL, 4, InvalidOid, NULL, NULL); + + newfcinfo->args[0].value = PG_GETARG_OID(0); + newfcinfo->args[0].isnull = PG_ARGISNULL(0); + newfcinfo->args[1].value = DEFAULT_RELPAGES; + newfcinfo->args[1].isnull = false; + newfcinfo->args[2].value = DEFAULT_RELTUPLES; + newfcinfo->args[2].isnull = false; + newfcinfo->args[3].value = DEFAULT_RELALLVISIBLE; + newfcinfo->args[3].isnull = false; + + PG_RETURN_BOOL(relation_statistics_update(newfcinfo, ERROR)); +} diff --git a/src/backend/statistics/stat_utils.c b/src/backend/statistics/stat_utils.c new file mode 100644 index 00000000000..4babed2e5d7 --- /dev/null +++ b/src/backend/statistics/stat_utils.c @@ -0,0 +1,94 @@ +/*------------------------------------------------------------------------- + * stat_utils.c + * + * PostgreSQL statistics manipulation utilities. + * + * Code supporting the direct manipulation of statistics. + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/statistics/stat_utils.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "access/relation.h" +#include "catalog/pg_database.h" +#include "miscadmin.h" +#include "statistics/stat_utils.h" +#include "utils/array.h" +#include "utils/builtins.h" +#include "utils/acl.h" +#include "utils/rel.h" + +/* + * Ensure that a given argument is not null. + */ +void +stats_check_required_arg(FunctionCallInfo fcinfo, + struct StatsArgInfo *arginfo, + int argnum) +{ + if (PG_ARGISNULL(argnum)) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("\"%s\" cannot be NULL", + arginfo[argnum].argname))); +} + +/* + * Lock relation in ShareUpdateExclusive mode, check privileges, and close the + * relation (but retain the lock). + * + * A role has privileges to set statistics on the relation if any of the + * following are true: + * - the role owns the current database and the relation is not shared + * - the role has the MAINTAIN privilege on the relation + */ +void +stats_lock_check_privileges(Oid reloid) +{ + Relation rel = relation_open(reloid, ShareUpdateExclusiveLock); + const char relkind = rel->rd_rel->relkind; + + /* All of the types that can be used with ANALYZE, plus indexes */ + switch (relkind) + { + case RELKIND_RELATION: + case RELKIND_INDEX: + case RELKIND_MATVIEW: + case RELKIND_FOREIGN_TABLE: + case RELKIND_PARTITIONED_TABLE: + case RELKIND_PARTITIONED_INDEX: + break; + default: + ereport(ERROR, + (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("cannot modify statistics for relation \"%s\"", + RelationGetRelationName(rel)), + errdetail_relkind_not_supported(rel->rd_rel->relkind))); + } + + if (rel->rd_rel->relisshared) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot modify statistics for shared relation"))); + + if (!object_ownercheck(DatabaseRelationId, MyDatabaseId, GetUserId())) + { + AclResult aclresult = pg_class_aclcheck(RelationGetRelid(rel), + GetUserId(), + ACL_MAINTAIN); + + if (aclresult != ACLCHECK_OK) + aclcheck_error(aclresult, + get_relkind_objtype(rel->rd_rel->relkind), + NameStr(rel->rd_rel->relname)); + } + + relation_close(rel, NoLock); +} diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 6a5476d3c4c..5999b4267f8 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -12184,5 +12184,19 @@ proallargtypes => '{int8,pg_lsn,pg_lsn,int4}', proargmodes => '{o,o,o,o}', proargnames => '{summarized_tli,summarized_lsn,pending_lsn,summarizer_pid}', prosrc => 'pg_get_wal_summarizer_state' }, - + # Statistics Import +{ oid => '9944', + descr => 'set statistics on relation', + proname => 'pg_set_relation_stats', provolatile => 'v', proisstrict => 'f', + proparallel => 'u', prorettype => 'bool', + proargtypes => 'regclass int4 float4 int4', + proargnames => '{relation,relpages,reltuples,relallvisible}', + prosrc => 'pg_set_relation_stats' }, +{ oid => '9945', + descr => 'clear statistics on relation', + proname => 'pg_clear_relation_stats', provolatile => 'v', proisstrict => 'f', + proparallel => 'u', prorettype => 'bool', + proargtypes => 'regclass', + proargnames => '{relation}', + prosrc => 'pg_clear_relation_stats' }, ] diff --git a/src/include/statistics/stat_utils.h b/src/include/statistics/stat_utils.h new file mode 100644 index 00000000000..46057c60c03 --- /dev/null +++ b/src/include/statistics/stat_utils.h @@ -0,0 +1,29 @@ +/*------------------------------------------------------------------------- + * + * stat_utils.h + * Extended statistics and selectivity estimation functions. + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/statistics/stat_utils.h + * + *------------------------------------------------------------------------- + */ +#ifndef STATS_UTILS_H +#define STATS_UTILS_H + +#include "fmgr.h" + +struct StatsArgInfo +{ + const char *argname; + Oid argtype; +}; + +extern void stats_check_required_arg(FunctionCallInfo fcinfo, + struct StatsArgInfo *arginfo, + int argnum); +extern void stats_lock_check_privileges(Oid reloid); + +#endif /* STATS_UTILS_H */ diff --git a/src/test/regress/expected/stats_import.out b/src/test/regress/expected/stats_import.out new file mode 100644 index 00000000000..cd1b80aa43a --- /dev/null +++ b/src/test/regress/expected/stats_import.out @@ -0,0 +1,143 @@ +CREATE SCHEMA stats_import; +CREATE TYPE stats_import.complex_type AS ( + a integer, + b real, + c text, + d date, + e jsonb); +CREATE TABLE stats_import.test( + id INTEGER PRIMARY KEY, + name text, + comp stats_import.complex_type, + arange int4range, + tags text[] +); +-- starting stats +SELECT relpages, reltuples, relallvisible +FROM pg_class +WHERE oid = 'stats_import.test'::regclass; + relpages | reltuples | relallvisible +----------+-----------+--------------- + 0 | -1 | 0 +(1 row) + +-- error: regclass not found +SELECT + pg_catalog.pg_set_relation_stats( + relation => 0::Oid, + relpages => 17::integer, + reltuples => 400.0::real, + relallvisible => 4::integer); +ERROR: could not open relation with OID 0 +-- relpages default +SELECT + pg_catalog.pg_set_relation_stats( + relation => 'stats_import.test'::regclass, + relpages => NULL::integer, + reltuples => 400.0::real, + relallvisible => 4::integer); + pg_set_relation_stats +----------------------- + t +(1 row) + +-- reltuples default +SELECT + pg_catalog.pg_set_relation_stats( + relation => 'stats_import.test'::regclass, + relpages => 17::integer, + reltuples => NULL::real, + relallvisible => 4::integer); + pg_set_relation_stats +----------------------- + t +(1 row) + +-- relallvisible default +SELECT + pg_catalog.pg_set_relation_stats( + relation => 'stats_import.test'::regclass, + relpages => 17::integer, + reltuples => 400.0::real, + relallvisible => NULL::integer); + pg_set_relation_stats +----------------------- + f +(1 row) + +-- named arguments +SELECT + pg_catalog.pg_set_relation_stats( + relation => 'stats_import.test'::regclass, + relpages => 17::integer, + reltuples => 400.0::real, + relallvisible => 4::integer); + pg_set_relation_stats +----------------------- + f +(1 row) + +SELECT relpages, reltuples, relallvisible +FROM pg_class +WHERE oid = 'stats_import.test'::regclass; + relpages | reltuples | relallvisible +----------+-----------+--------------- + 17 | 400 | 4 +(1 row) + +-- positional arguments +SELECT + pg_catalog.pg_set_relation_stats( + 'stats_import.test'::regclass, + 18::integer, + 401.0::real, + 5::integer); + pg_set_relation_stats +----------------------- + t +(1 row) + +SELECT relpages, reltuples, relallvisible +FROM pg_class +WHERE oid = 'stats_import.test'::regclass; + relpages | reltuples | relallvisible +----------+-----------+--------------- + 18 | 401 | 5 +(1 row) + +-- clear +SELECT + pg_catalog.pg_clear_relation_stats( + 'stats_import.test'::regclass); + pg_clear_relation_stats +------------------------- + t +(1 row) + +SELECT relpages, reltuples, relallvisible +FROM pg_class +WHERE oid = 'stats_import.test'::regclass; + relpages | reltuples | relallvisible +----------+-----------+--------------- + 0 | -1 | 0 +(1 row) + +-- invalid relkinds for statistics +CREATE SEQUENCE stats_import.testseq; +CREATE VIEW stats_import.testview AS SELECT * FROM stats_import.test; +SELECT + pg_catalog.pg_clear_relation_stats( + 'stats_import.testseq'::regclass); +ERROR: cannot modify statistics for relation "testseq" +DETAIL: This operation is not supported for sequences. +SELECT + pg_catalog.pg_clear_relation_stats( + 'stats_import.testview'::regclass); +ERROR: cannot modify statistics for relation "testview" +DETAIL: This operation is not supported for views. +DROP SCHEMA stats_import CASCADE; +NOTICE: drop cascades to 4 other objects +DETAIL: drop cascades to type stats_import.complex_type +drop cascades to table stats_import.test +drop cascades to sequence stats_import.testseq +drop cascades to view stats_import.testview diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule index 7b36f7c183b..be9f5736c44 100644 --- a/src/test/regress/parallel_schedule +++ b/src/test/regress/parallel_schedule @@ -28,7 +28,7 @@ test: strings md5 numerology point lseg line box path polygon circle date time t # geometry depends on point, lseg, line, box, path, polygon, circle # horology depends on date, time, timetz, timestamp, timestamptz, interval # ---------- -test: geometry horology tstypes regex type_sanity opr_sanity misc_sanity comments expressions unicode xid mvcc database encoding euc_kr +test: geometry horology tstypes regex type_sanity opr_sanity misc_sanity comments expressions unicode xid mvcc database encoding euc_kr stats_import # ---------- # Load huge amounts of data diff --git a/src/test/regress/sql/stats_import.sql b/src/test/regress/sql/stats_import.sql new file mode 100644 index 00000000000..3e9f6d91241 --- /dev/null +++ b/src/test/regress/sql/stats_import.sql @@ -0,0 +1,98 @@ +CREATE SCHEMA stats_import; + +CREATE TYPE stats_import.complex_type AS ( + a integer, + b real, + c text, + d date, + e jsonb); + +CREATE TABLE stats_import.test( + id INTEGER PRIMARY KEY, + name text, + comp stats_import.complex_type, + arange int4range, + tags text[] +); + +-- starting stats +SELECT relpages, reltuples, relallvisible +FROM pg_class +WHERE oid = 'stats_import.test'::regclass; + +-- error: regclass not found +SELECT + pg_catalog.pg_set_relation_stats( + relation => 0::Oid, + relpages => 17::integer, + reltuples => 400.0::real, + relallvisible => 4::integer); + +-- relpages default +SELECT + pg_catalog.pg_set_relation_stats( + relation => 'stats_import.test'::regclass, + relpages => NULL::integer, + reltuples => 400.0::real, + relallvisible => 4::integer); + +-- reltuples default +SELECT + pg_catalog.pg_set_relation_stats( + relation => 'stats_import.test'::regclass, + relpages => 17::integer, + reltuples => NULL::real, + relallvisible => 4::integer); + +-- relallvisible default +SELECT + pg_catalog.pg_set_relation_stats( + relation => 'stats_import.test'::regclass, + relpages => 17::integer, + reltuples => 400.0::real, + relallvisible => NULL::integer); + +-- named arguments +SELECT + pg_catalog.pg_set_relation_stats( + relation => 'stats_import.test'::regclass, + relpages => 17::integer, + reltuples => 400.0::real, + relallvisible => 4::integer); + +SELECT relpages, reltuples, relallvisible +FROM pg_class +WHERE oid = 'stats_import.test'::regclass; + +-- positional arguments +SELECT + pg_catalog.pg_set_relation_stats( + 'stats_import.test'::regclass, + 18::integer, + 401.0::real, + 5::integer); + +SELECT relpages, reltuples, relallvisible +FROM pg_class +WHERE oid = 'stats_import.test'::regclass; + +-- clear +SELECT + pg_catalog.pg_clear_relation_stats( + 'stats_import.test'::regclass); + +SELECT relpages, reltuples, relallvisible +FROM pg_class +WHERE oid = 'stats_import.test'::regclass; + +-- invalid relkinds for statistics +CREATE SEQUENCE stats_import.testseq; +CREATE VIEW stats_import.testview AS SELECT * FROM stats_import.test; +SELECT + pg_catalog.pg_clear_relation_stats( + 'stats_import.testseq'::regclass); +SELECT + pg_catalog.pg_clear_relation_stats( + 'stats_import.testview'::regclass); + +DROP SCHEMA stats_import CASCADE; From 931d841af1cf8dff9bfecb14300615a17630bb9f Mon Sep 17 00:00:00 2001 From: Jeff Davis Date: Fri, 18 Oct 2024 10:44:15 -0700 Subject: [PATCH 04/54] Allow pg_set_relation_stats() to set relpages to -1. While the default value for relpages is 0, if a partitioned table with at least one child has been analyzed, then the partititoned table will have a relpages value of -1. Author: Corey Huinker Discussion: https://postgr.es/m/CADkLM=fajh1Lpcyr_XsMmq-9Z=SGk-u+_Zeac7Pt0RAN3uiVCg@mail.gmail.com --- doc/src/sgml/func.sgml | 2 +- src/backend/statistics/relation_stats.c | 5 +++ src/test/regress/expected/stats_import.out | 38 +++++++++++++++++++++- src/test/regress/sql/stats_import.sql | 25 ++++++++++++++ 4 files changed, 68 insertions(+), 2 deletions(-) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 3a83b989f83..1ac4c7d0deb 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -30012,7 +30012,7 @@ postgres=# SELECT '0/0'::pg_lsn + pd.segment_number * ps.setting::int + :offset The value of relpages must be greater than - or equal to 0, + or equal to -1, reltuples must be greater than or equal to -1.0, and relallvisible must be greater than or equal to 0. diff --git a/src/backend/statistics/relation_stats.c b/src/backend/statistics/relation_stats.c index ffa3d83a878..1a6d1640c30 100644 --- a/src/backend/statistics/relation_stats.c +++ b/src/backend/statistics/relation_stats.c @@ -99,6 +99,11 @@ relation_statistics_update(FunctionCallInfo fcinfo, int elevel) { int32 relpages = PG_GETARG_INT32(RELPAGES_ARG); + /* + * Partitioned tables may have relpages=-1. Note: for relations with + * no storage, relpages=-1 is not used consistently, but must be + * supported here. + */ if (relpages < -1) { ereport(elevel, diff --git a/src/test/regress/expected/stats_import.out b/src/test/regress/expected/stats_import.out index cd1b80aa43a..495e9b03485 100644 --- a/src/test/regress/expected/stats_import.out +++ b/src/test/regress/expected/stats_import.out @@ -135,9 +135,45 @@ SELECT 'stats_import.testview'::regclass); ERROR: cannot modify statistics for relation "testview" DETAIL: This operation is not supported for views. +-- relpages may be -1 for partitioned tables +CREATE TABLE stats_import.part_parent ( i integer ) PARTITION BY RANGE(i); +CREATE TABLE stats_import.part_child_1 + PARTITION OF stats_import.part_parent + FOR VALUES FROM (0) TO (10); +ANALYZE stats_import.part_parent; +SELECT relpages +FROM pg_class +WHERE oid = 'stats_import.part_parent'::regclass; + relpages +---------- + -1 +(1 row) + +-- although partitioned tables have no storage, setting relpages to a +-- positive value is still allowed +SELECT + pg_catalog.pg_set_relation_stats( + relation => 'stats_import.part_parent'::regclass, + relpages => 2::integer); + pg_set_relation_stats +----------------------- + t +(1 row) + +-- nothing stops us from setting it to -1 +SELECT + pg_catalog.pg_set_relation_stats( + relation => 'stats_import.part_parent'::regclass, + relpages => -1::integer); + pg_set_relation_stats +----------------------- + t +(1 row) + DROP SCHEMA stats_import CASCADE; -NOTICE: drop cascades to 4 other objects +NOTICE: drop cascades to 5 other objects DETAIL: drop cascades to type stats_import.complex_type drop cascades to table stats_import.test drop cascades to sequence stats_import.testseq drop cascades to view stats_import.testview +drop cascades to table stats_import.part_parent diff --git a/src/test/regress/sql/stats_import.sql b/src/test/regress/sql/stats_import.sql index 3e9f6d91241..108b0c45586 100644 --- a/src/test/regress/sql/stats_import.sql +++ b/src/test/regress/sql/stats_import.sql @@ -95,4 +95,29 @@ SELECT pg_catalog.pg_clear_relation_stats( 'stats_import.testview'::regclass); +-- relpages may be -1 for partitioned tables +CREATE TABLE stats_import.part_parent ( i integer ) PARTITION BY RANGE(i); +CREATE TABLE stats_import.part_child_1 + PARTITION OF stats_import.part_parent + FOR VALUES FROM (0) TO (10); + +ANALYZE stats_import.part_parent; + +SELECT relpages +FROM pg_class +WHERE oid = 'stats_import.part_parent'::regclass; + +-- although partitioned tables have no storage, setting relpages to a +-- positive value is still allowed +SELECT + pg_catalog.pg_set_relation_stats( + relation => 'stats_import.part_parent'::regclass, + relpages => 2::integer); + +-- nothing stops us from setting it to -1 +SELECT + pg_catalog.pg_set_relation_stats( + relation => 'stats_import.part_parent'::regclass, + relpages => -1::integer); + DROP SCHEMA stats_import CASCADE; From 05f8dca992253ff76bcf128d94bf339267f3e3d7 Mon Sep 17 00:00:00 2001 From: Jeff Davis Date: Fri, 18 Oct 2024 10:57:46 -0700 Subject: [PATCH 05/54] Disable autovacuum for tables in stats import tests. While we haven't observed any test instability, it seems like a good idea to disable autovacuum during the stats import tests. Author: Corey Huinker Discussion: https://postgr.es/m/CADkLM=fajh1Lpcyr_XsMmq-9Z=SGk-u+_Zeac7Pt0RAN3uiVCg@mail.gmail.com --- src/test/regress/expected/stats_import.out | 5 +++-- src/test/regress/sql/stats_import.sql | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/test/regress/expected/stats_import.out b/src/test/regress/expected/stats_import.out index 495e9b03485..a4e3a0f3c48 100644 --- a/src/test/regress/expected/stats_import.out +++ b/src/test/regress/expected/stats_import.out @@ -11,7 +11,7 @@ CREATE TABLE stats_import.test( comp stats_import.complex_type, arange int4range, tags text[] -); +) WITH (autovacuum_enabled = false); -- starting stats SELECT relpages, reltuples, relallvisible FROM pg_class @@ -139,7 +139,8 @@ DETAIL: This operation is not supported for views. CREATE TABLE stats_import.part_parent ( i integer ) PARTITION BY RANGE(i); CREATE TABLE stats_import.part_child_1 PARTITION OF stats_import.part_parent - FOR VALUES FROM (0) TO (10); + FOR VALUES FROM (0) TO (10) + WITH (autovacuum_enabled = false); ANALYZE stats_import.part_parent; SELECT relpages FROM pg_class diff --git a/src/test/regress/sql/stats_import.sql b/src/test/regress/sql/stats_import.sql index 108b0c45586..126b3ab9b9c 100644 --- a/src/test/regress/sql/stats_import.sql +++ b/src/test/regress/sql/stats_import.sql @@ -13,7 +13,7 @@ CREATE TABLE stats_import.test( comp stats_import.complex_type, arange int4range, tags text[] -); +) WITH (autovacuum_enabled = false); -- starting stats SELECT relpages, reltuples, relallvisible @@ -99,7 +99,8 @@ SELECT CREATE TABLE stats_import.part_parent ( i integer ) PARTITION BY RANGE(i); CREATE TABLE stats_import.part_child_1 PARTITION OF stats_import.part_parent - FOR VALUES FROM (0) TO (10); + FOR VALUES FROM (0) TO (10) + WITH (autovacuum_enabled = false); ANALYZE stats_import.part_parent; From 718078a5a74b7a2b646465e750f974a95f9c4c11 Mon Sep 17 00:00:00 2001 From: Jeff Davis Date: Tue, 22 Oct 2024 12:48:01 -0700 Subject: [PATCH 06/54] Change pg_*_relation_stats() functions to return type to void. These functions will either raise an ERROR or run to normal completion, so no return value is necessary. Bump catalog version. Author: Corey Huinker Discussion: https://postgr.es/m/CADkLM=cBF8rnphuTyHFi3KYzB9ByDgx57HwK9Rz2yp7S+Om87w@mail.gmail.com --- doc/src/sgml/func.sgml | 11 ++++------- src/backend/catalog/system_functions.sql | 2 +- src/backend/statistics/relation_stats.c | 6 ++++-- src/include/catalog/pg_proc.dat | 4 ++-- src/test/regress/expected/stats_import.out | 12 ++++++------ 5 files changed, 17 insertions(+), 18 deletions(-) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 1ac4c7d0deb..8bc27b66336 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -29989,15 +29989,13 @@ postgres=# SELECT '0/0'::pg_lsn + pd.segment_number * ps.setting::int + :offset , relpages integer , reltuples real , relallvisible integer ) - boolean + void Updates relation-level statistics for the given relation to the specified values. The parameters correspond to columns in pg_class. Unspecified - or NULL values leave the setting - unchanged. Returns true if a change was made; - false otherwise. + or NULL values leave the setting unchanged. Ordinarily, these statistics are collected automatically or updated @@ -30027,12 +30025,11 @@ postgres=# SELECT '0/0'::pg_lsn + pd.segment_number * ps.setting::int + :offset pg_clear_relation_stats pg_clear_relation_stats ( relation regclass ) - boolean + void Clears table-level statistics for the given relation, as though the - table was newly created. Returns true if a change - was made; false otherwise. + table was newly created. The caller must have the MAINTAIN privilege on diff --git a/src/backend/catalog/system_functions.sql b/src/backend/catalog/system_functions.sql index 0b13eec0ca8..2b7f3e20267 100644 --- a/src/backend/catalog/system_functions.sql +++ b/src/backend/catalog/system_functions.sql @@ -641,7 +641,7 @@ CREATE OR REPLACE FUNCTION relpages integer DEFAULT NULL, reltuples real DEFAULT NULL, relallvisible integer DEFAULT NULL) -RETURNS bool +RETURNS void LANGUAGE INTERNAL CALLED ON NULL INPUT VOLATILE AS 'pg_set_relation_stats'; diff --git a/src/backend/statistics/relation_stats.c b/src/backend/statistics/relation_stats.c index 1a6d1640c30..b1eb8a9bbaf 100644 --- a/src/backend/statistics/relation_stats.c +++ b/src/backend/statistics/relation_stats.c @@ -188,7 +188,8 @@ relation_statistics_update(FunctionCallInfo fcinfo, int elevel) Datum pg_set_relation_stats(PG_FUNCTION_ARGS) { - PG_RETURN_BOOL(relation_statistics_update(fcinfo, ERROR)); + relation_statistics_update(fcinfo, ERROR); + PG_RETURN_VOID(); } /* @@ -211,5 +212,6 @@ pg_clear_relation_stats(PG_FUNCTION_ARGS) newfcinfo->args[3].value = DEFAULT_RELALLVISIBLE; newfcinfo->args[3].isnull = false; - PG_RETURN_BOOL(relation_statistics_update(newfcinfo, ERROR)); + relation_statistics_update(newfcinfo, ERROR); + PG_RETURN_VOID(); } diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 5999b4267f8..76e4634ea4e 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -12188,14 +12188,14 @@ { oid => '9944', descr => 'set statistics on relation', proname => 'pg_set_relation_stats', provolatile => 'v', proisstrict => 'f', - proparallel => 'u', prorettype => 'bool', + proparallel => 'u', prorettype => 'void', proargtypes => 'regclass int4 float4 int4', proargnames => '{relation,relpages,reltuples,relallvisible}', prosrc => 'pg_set_relation_stats' }, { oid => '9945', descr => 'clear statistics on relation', proname => 'pg_clear_relation_stats', provolatile => 'v', proisstrict => 'f', - proparallel => 'u', prorettype => 'bool', + proparallel => 'u', prorettype => 'void', proargtypes => 'regclass', proargnames => '{relation}', prosrc => 'pg_clear_relation_stats' }, diff --git a/src/test/regress/expected/stats_import.out b/src/test/regress/expected/stats_import.out index a4e3a0f3c48..df39ac2b635 100644 --- a/src/test/regress/expected/stats_import.out +++ b/src/test/regress/expected/stats_import.out @@ -38,7 +38,7 @@ SELECT relallvisible => 4::integer); pg_set_relation_stats ----------------------- - t + (1 row) -- reltuples default @@ -50,7 +50,7 @@ SELECT relallvisible => 4::integer); pg_set_relation_stats ----------------------- - t + (1 row) -- relallvisible default @@ -62,7 +62,7 @@ SELECT relallvisible => NULL::integer); pg_set_relation_stats ----------------------- - f + (1 row) -- named arguments @@ -74,7 +74,7 @@ SELECT relallvisible => 4::integer); pg_set_relation_stats ----------------------- - f + (1 row) SELECT relpages, reltuples, relallvisible @@ -94,7 +94,7 @@ SELECT 5::integer); pg_set_relation_stats ----------------------- - t + (1 row) SELECT relpages, reltuples, relallvisible @@ -111,7 +111,7 @@ SELECT 'stats_import.test'::regclass); pg_clear_relation_stats ------------------------- - t + (1 row) SELECT relpages, reltuples, relallvisible From 5a64e1950d93a1b1bf7e3b4c582e71b502b99362 Mon Sep 17 00:00:00 2001 From: Jeff Davis Date: Tue, 22 Oct 2024 15:06:55 -0700 Subject: [PATCH 07/54] Add functions pg_set_attribute_stats() and pg_clear_attribute_stats(). Enable manipulation of attribute statistics. Only superficial validation is performed, so it's possible to add nonsense, and it's up to the planner (or other users of statistics) to behave reasonably in that case. Bump catalog version. Author: Corey Huinker Discussion: https://postgr.es/m/CADkLM=eErgzn7ECDpwFcptJKOk9SxZEk5Pot4d94eVTZsvj3gw@mail.gmail.com --- doc/src/sgml/func.sgml | 72 ++ src/backend/catalog/system_functions.sql | 22 + src/backend/statistics/Makefile | 1 + src/backend/statistics/attribute_stats.c | 869 +++++++++++++++++++++ src/backend/statistics/meson.build | 1 + src/backend/statistics/stat_utils.c | 73 ++ src/include/catalog/pg_proc.dat | 14 + src/include/statistics/stat_utils.h | 7 + src/test/regress/expected/stats_import.out | 663 +++++++++++++++- src/test/regress/sql/stats_import.sql | 545 +++++++++++++ 10 files changed, 2264 insertions(+), 3 deletions(-) create mode 100644 src/backend/statistics/attribute_stats.c diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 8bc27b66336..817669ed896 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -30037,6 +30037,78 @@ postgres=# SELECT '0/0'::pg_lsn + pd.segment_number * ps.setting::int + :offset + + + + + + pg_set_attribute_stats + + pg_set_attribute_stats ( + relation regclass, + attname name, + inherited boolean + , null_frac real + , avg_width integer + , n_distinct real + , most_common_vals text, most_common_freqs real[] + , histogram_bounds text + , correlation real + , most_common_elems text, most_common_elem_freqs real[] + , elem_count_histogram real[] + , range_length_histogram text + , range_empty_frac real + , range_bounds_histogram text ) + void + + + Creates or updates attribute-level statistics for the given relation + and attribute name to the specified values. The parameters correspond + to to attributes of the same name found in the pg_stats + view. + + + Optional parameters default to NULL, which leave + the corresponding statistic unchanged. + + + Ordinarily, these statistics are collected automatically or updated + as a part of or , so it's not necessary to call this + function. However, it may be useful when testing the effects of + statistics on the planner to understand or anticipate plan changes. + + + The caller must have the MAINTAIN privilege on + the table or be the owner of the database. + + + + + + + + + pg_clear_attribute_stats + + pg_clear_attribute_stats ( + relation regclass, + attname name, + inherited boolean ) + boolean + + + Clears table-level statistics for the given relation attribute, as + though the table was newly created. + + + The caller must have the MAINTAIN privilege on + the table or be the owner of the database. + + + + diff --git a/src/backend/catalog/system_functions.sql b/src/backend/catalog/system_functions.sql index 2b7f3e20267..05227c03097 100644 --- a/src/backend/catalog/system_functions.sql +++ b/src/backend/catalog/system_functions.sql @@ -646,6 +646,28 @@ LANGUAGE INTERNAL CALLED ON NULL INPUT VOLATILE AS 'pg_set_relation_stats'; +CREATE OR REPLACE FUNCTION + pg_set_attribute_stats(relation regclass, + attname name, + inherited bool, + null_frac real DEFAULT NULL, + avg_width integer DEFAULT NULL, + n_distinct real DEFAULT NULL, + most_common_vals text DEFAULT NULL, + most_common_freqs real[] DEFAULT NULL, + histogram_bounds text DEFAULT NULL, + correlation real DEFAULT NULL, + most_common_elems text DEFAULT NULL, + most_common_elem_freqs real[] DEFAULT NULL, + elem_count_histogram real[] DEFAULT NULL, + range_length_histogram text DEFAULT NULL, + range_empty_frac real DEFAULT NULL, + range_bounds_histogram text DEFAULT NULL) +RETURNS void +LANGUAGE INTERNAL +CALLED ON NULL INPUT VOLATILE +AS 'pg_set_attribute_stats'; + -- -- The default permissions for functions mean that anyone can execute them. -- A number of functions shouldn't be executable by just anyone, but rather diff --git a/src/backend/statistics/Makefile b/src/backend/statistics/Makefile index 041f5f8a581..4672bd90f22 100644 --- a/src/backend/statistics/Makefile +++ b/src/backend/statistics/Makefile @@ -13,6 +13,7 @@ top_builddir = ../../.. include $(top_builddir)/src/Makefile.global OBJS = \ + attribute_stats.o \ dependencies.o \ extended_stats.o \ mcv.o \ diff --git a/src/backend/statistics/attribute_stats.c b/src/backend/statistics/attribute_stats.c new file mode 100644 index 00000000000..c920409680a --- /dev/null +++ b/src/backend/statistics/attribute_stats.c @@ -0,0 +1,869 @@ +/*------------------------------------------------------------------------- + * attribute_stats.c + * + * PostgreSQL relation attribute statistics manipulation. + * + * Code supporting the direct import of relation attribute statistics, similar + * to what is done by the ANALYZE command. + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/statistics/attribute_stats.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "access/heapam.h" +#include "catalog/indexing.h" +#include "catalog/pg_collation.h" +#include "catalog/pg_operator.h" +#include "nodes/nodeFuncs.h" +#include "statistics/statistics.h" +#include "statistics/stat_utils.h" +#include "utils/array.h" +#include "utils/builtins.h" +#include "utils/fmgroids.h" +#include "utils/lsyscache.h" +#include "utils/syscache.h" + +#define DEFAULT_NULL_FRAC Float4GetDatum(0.0) +#define DEFAULT_AVG_WIDTH Int32GetDatum(0) /* unknown */ +#define DEFAULT_N_DISTINCT Float4GetDatum(0.0) /* unknown */ + +enum attribute_stats_argnum +{ + ATTRELATION_ARG = 0, + ATTNAME_ARG, + INHERITED_ARG, + NULL_FRAC_ARG, + AVG_WIDTH_ARG, + N_DISTINCT_ARG, + MOST_COMMON_VALS_ARG, + MOST_COMMON_FREQS_ARG, + HISTOGRAM_BOUNDS_ARG, + CORRELATION_ARG, + MOST_COMMON_ELEMS_ARG, + MOST_COMMON_ELEM_FREQS_ARG, + ELEM_COUNT_HISTOGRAM_ARG, + RANGE_LENGTH_HISTOGRAM_ARG, + RANGE_EMPTY_FRAC_ARG, + RANGE_BOUNDS_HISTOGRAM_ARG, + NUM_ATTRIBUTE_STATS_ARGS +}; + +static struct StatsArgInfo attarginfo[] = +{ + [ATTRELATION_ARG] = {"relation", REGCLASSOID}, + [ATTNAME_ARG] = {"attname", NAMEOID}, + [INHERITED_ARG] = {"inherited", BOOLOID}, + [NULL_FRAC_ARG] = {"null_frac", FLOAT4OID}, + [AVG_WIDTH_ARG] = {"avg_width", INT4OID}, + [N_DISTINCT_ARG] = {"n_distinct", FLOAT4OID}, + [MOST_COMMON_VALS_ARG] = {"most_common_vals", TEXTOID}, + [MOST_COMMON_FREQS_ARG] = {"most_common_freqs", FLOAT4ARRAYOID}, + [HISTOGRAM_BOUNDS_ARG] = {"histogram_bounds", TEXTOID}, + [CORRELATION_ARG] = {"correlation", FLOAT4OID}, + [MOST_COMMON_ELEMS_ARG] = {"most_common_elems", TEXTOID}, + [MOST_COMMON_ELEM_FREQS_ARG] = {"most_common_elem_freqs", FLOAT4ARRAYOID}, + [ELEM_COUNT_HISTOGRAM_ARG] = {"elem_count_histogram", FLOAT4ARRAYOID}, + [RANGE_LENGTH_HISTOGRAM_ARG] = {"range_length_histogram", TEXTOID}, + [RANGE_EMPTY_FRAC_ARG] = {"range_empty_frac", FLOAT4OID}, + [RANGE_BOUNDS_HISTOGRAM_ARG] = {"range_bounds_histogram", TEXTOID}, + [NUM_ATTRIBUTE_STATS_ARGS] = {0} +}; + +static bool attribute_statistics_update(FunctionCallInfo fcinfo, int elevel); +static Node *get_attr_expr(Relation rel, int attnum); +static void get_attr_stat_type(Oid reloid, AttrNumber attnum, int elevel, + Oid *atttypid, int32 *atttypmod, + char *atttyptype, Oid *atttypcoll, + Oid *eq_opr, Oid *lt_opr); +static bool get_elem_stat_type(Oid atttypid, char atttyptype, int elevel, + Oid *elemtypid, Oid *elem_eq_opr); +static Datum text_to_stavalues(const char *staname, FmgrInfo *array_in, Datum d, + Oid typid, int32 typmod, int elevel, bool *ok); +static void set_stats_slot(Datum *values, bool *nulls, bool *replaces, + int16 stakind, Oid staop, Oid stacoll, + Datum stanumbers, bool stanumbers_isnull, + Datum stavalues, bool stavalues_isnull); +static void upsert_pg_statistic(Relation starel, HeapTuple oldtup, + Datum *values, bool *nulls, bool *replaces); +static bool delete_pg_statistic(Oid reloid, AttrNumber attnum, bool stainherit); +static void init_empty_stats_tuple(Oid reloid, int16 attnum, bool inherited, + Datum *values, bool *nulls, bool *replaces); + +/* + * Insert or Update Attribute Statistics + * + * See pg_statistic.h for an explanation of how each statistic kind is + * stored. Custom statistics kinds are not supported. + * + * Depending on the statistics kind, we need to derive information from the + * attribute for which we're storing the stats. For instance, the MCVs are + * stored as an anyarray, and the representation of the array needs to store + * the correct element type, which must be derived from the attribute. + * + * Major errors, such as the table not existing, the attribute not existing, + * or a permissions failure are always reported at ERROR. Other errors, such + * as a conversion failure on one statistic kind, are reported at 'elevel', + * and other statistic kinds may still be updated. + */ +static bool +attribute_statistics_update(FunctionCallInfo fcinfo, int elevel) +{ + Oid reloid; + Name attname; + bool inherited; + AttrNumber attnum; + + Relation starel; + HeapTuple statup; + + Oid atttypid = InvalidOid; + int32 atttypmod; + char atttyptype; + Oid atttypcoll = InvalidOid; + Oid eq_opr = InvalidOid; + Oid lt_opr = InvalidOid; + + Oid elemtypid = InvalidOid; + Oid elem_eq_opr = InvalidOid; + + FmgrInfo array_in_fn; + + bool do_mcv = !PG_ARGISNULL(MOST_COMMON_FREQS_ARG) && + !PG_ARGISNULL(MOST_COMMON_VALS_ARG); + bool do_histogram = !PG_ARGISNULL(HISTOGRAM_BOUNDS_ARG); + bool do_correlation = !PG_ARGISNULL(CORRELATION_ARG); + bool do_mcelem = !PG_ARGISNULL(MOST_COMMON_ELEMS_ARG) && + !PG_ARGISNULL(MOST_COMMON_ELEM_FREQS_ARG); + bool do_dechist = !PG_ARGISNULL(ELEM_COUNT_HISTOGRAM_ARG); + bool do_bounds_histogram = !PG_ARGISNULL(RANGE_BOUNDS_HISTOGRAM_ARG); + bool do_range_length_histogram = !PG_ARGISNULL(RANGE_LENGTH_HISTOGRAM_ARG) && + !PG_ARGISNULL(RANGE_EMPTY_FRAC_ARG); + + Datum values[Natts_pg_statistic] = {0}; + bool nulls[Natts_pg_statistic] = {0}; + bool replaces[Natts_pg_statistic] = {0}; + + bool result = true; + + stats_check_required_arg(fcinfo, attarginfo, ATTRELATION_ARG); + reloid = PG_GETARG_OID(ATTRELATION_ARG); + + /* lock before looking up attribute */ + stats_lock_check_privileges(reloid); + + stats_check_required_arg(fcinfo, attarginfo, ATTNAME_ARG); + attname = PG_GETARG_NAME(ATTNAME_ARG); + attnum = get_attnum(reloid, NameStr(*attname)); + + stats_check_required_arg(fcinfo, attarginfo, INHERITED_ARG); + inherited = PG_GETARG_BOOL(INHERITED_ARG); + + /* + * Check argument sanity. If some arguments are unusable, emit at elevel + * and set the corresponding argument to NULL in fcinfo. + */ + + if (!stats_check_arg_array(fcinfo, attarginfo, MOST_COMMON_FREQS_ARG, + elevel)) + { + do_mcv = false; + result = false; + } + + if (!stats_check_arg_array(fcinfo, attarginfo, MOST_COMMON_ELEM_FREQS_ARG, + elevel)) + { + do_mcelem = false; + result = false; + } + if (!stats_check_arg_array(fcinfo, attarginfo, ELEM_COUNT_HISTOGRAM_ARG, + elevel)) + { + do_dechist = false; + result = false; + } + + if (!stats_check_arg_pair(fcinfo, attarginfo, + MOST_COMMON_VALS_ARG, MOST_COMMON_FREQS_ARG, + elevel)) + { + do_mcv = false; + result = false; + } + + if (!stats_check_arg_pair(fcinfo, attarginfo, + MOST_COMMON_ELEMS_ARG, + MOST_COMMON_ELEM_FREQS_ARG, elevel)) + { + do_mcelem = false; + result = false; + } + + if (!stats_check_arg_pair(fcinfo, attarginfo, + RANGE_LENGTH_HISTOGRAM_ARG, + RANGE_EMPTY_FRAC_ARG, elevel)) + { + do_range_length_histogram = false; + result = false; + } + + /* derive information from attribute */ + get_attr_stat_type(reloid, attnum, elevel, + &atttypid, &atttypmod, + &atttyptype, &atttypcoll, + &eq_opr, <_opr); + + /* if needed, derive element type */ + if (do_mcelem || do_dechist) + { + if (!get_elem_stat_type(atttypid, atttyptype, elevel, + &elemtypid, &elem_eq_opr)) + { + ereport(elevel, + (errmsg("unable to determine element type of attribute \"%s\"", NameStr(*attname)), + errdetail("Cannot set STATISTIC_KIND_MCELEM or STATISTIC_KIND_DECHIST."))); + elemtypid = InvalidOid; + elem_eq_opr = InvalidOid; + + do_mcelem = false; + do_dechist = false; + result = false; + } + } + + /* histogram and correlation require less-than operator */ + if ((do_histogram || do_correlation) && !OidIsValid(lt_opr)) + { + ereport(elevel, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("could not determine less-than operator for attribute \"%s\"", NameStr(*attname)), + errdetail("Cannot set STATISTIC_KIND_HISTOGRAM or STATISTIC_KIND_CORRELATION."))); + + do_histogram = false; + do_correlation = false; + result = false; + } + + /* only range types can have range stats */ + if ((do_range_length_histogram || do_bounds_histogram) && + !(atttyptype == TYPTYPE_RANGE || atttyptype == TYPTYPE_MULTIRANGE)) + { + ereport(elevel, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("attribute \"%s\" is not a range type", NameStr(*attname)), + errdetail("Cannot set STATISTIC_KIND_RANGE_LENGTH_HISTOGRAM or STATISTIC_KIND_BOUNDS_HISTOGRAM."))); + + do_bounds_histogram = false; + do_range_length_histogram = false; + result = false; + } + + fmgr_info(F_ARRAY_IN, &array_in_fn); + + starel = table_open(StatisticRelationId, RowExclusiveLock); + + statup = SearchSysCache3(STATRELATTINH, reloid, attnum, inherited); + + /* initialize from existing tuple if exists */ + if (HeapTupleIsValid(statup)) + heap_deform_tuple(statup, RelationGetDescr(starel), values, nulls); + else + init_empty_stats_tuple(reloid, attnum, inherited, values, nulls, + replaces); + + /* if specified, set to argument values */ + if (!PG_ARGISNULL(NULL_FRAC_ARG)) + { + values[Anum_pg_statistic_stanullfrac - 1] = PG_GETARG_DATUM(NULL_FRAC_ARG); + replaces[Anum_pg_statistic_stanullfrac - 1] = true; + } + if (!PG_ARGISNULL(AVG_WIDTH_ARG)) + { + values[Anum_pg_statistic_stawidth - 1] = PG_GETARG_DATUM(AVG_WIDTH_ARG); + replaces[Anum_pg_statistic_stawidth - 1] = true; + } + if (!PG_ARGISNULL(N_DISTINCT_ARG)) + { + values[Anum_pg_statistic_stadistinct - 1] = PG_GETARG_DATUM(N_DISTINCT_ARG); + replaces[Anum_pg_statistic_stadistinct - 1] = true; + } + + /* STATISTIC_KIND_MCV */ + if (do_mcv) + { + bool converted; + Datum stanumbers = PG_GETARG_DATUM(MOST_COMMON_FREQS_ARG); + Datum stavalues = text_to_stavalues("most_common_vals", + &array_in_fn, + PG_GETARG_DATUM(MOST_COMMON_VALS_ARG), + atttypid, atttypmod, + elevel, &converted); + + if (converted) + { + set_stats_slot(values, nulls, replaces, + STATISTIC_KIND_MCV, + eq_opr, atttypcoll, + stanumbers, false, stavalues, false); + } + else + result = false; + } + + /* STATISTIC_KIND_HISTOGRAM */ + if (do_histogram) + { + Datum stavalues; + bool converted = false; + + stavalues = text_to_stavalues("histogram_bounds", + &array_in_fn, + PG_GETARG_DATUM(HISTOGRAM_BOUNDS_ARG), + atttypid, atttypmod, elevel, + &converted); + + if (converted) + { + set_stats_slot(values, nulls, replaces, + STATISTIC_KIND_HISTOGRAM, + lt_opr, atttypcoll, + 0, true, stavalues, false); + } + else + result = false; + } + + /* STATISTIC_KIND_CORRELATION */ + if (do_correlation) + { + Datum elems[] = {PG_GETARG_DATUM(CORRELATION_ARG)}; + ArrayType *arry = construct_array_builtin(elems, 1, FLOAT4OID); + Datum stanumbers = PointerGetDatum(arry); + + set_stats_slot(values, nulls, replaces, + STATISTIC_KIND_CORRELATION, + lt_opr, atttypcoll, + stanumbers, false, 0, true); + } + + /* STATISTIC_KIND_MCELEM */ + if (do_mcelem) + { + Datum stanumbers = PG_GETARG_DATUM(MOST_COMMON_ELEM_FREQS_ARG); + bool converted = false; + Datum stavalues; + + stavalues = text_to_stavalues("most_common_elems", + &array_in_fn, + PG_GETARG_DATUM(MOST_COMMON_ELEMS_ARG), + elemtypid, atttypmod, + elevel, &converted); + + if (converted) + { + set_stats_slot(values, nulls, replaces, + STATISTIC_KIND_MCELEM, + elem_eq_opr, atttypcoll, + stanumbers, false, stavalues, false); + } + else + result = false; + } + + /* STATISTIC_KIND_DECHIST */ + if (do_dechist) + { + Datum stanumbers = PG_GETARG_DATUM(ELEM_COUNT_HISTOGRAM_ARG); + + set_stats_slot(values, nulls, replaces, + STATISTIC_KIND_DECHIST, + elem_eq_opr, atttypcoll, + stanumbers, false, 0, true); + } + + /* + * STATISTIC_KIND_BOUNDS_HISTOGRAM + * + * This stakind appears before STATISTIC_KIND_RANGE_LENGTH_HISTOGRAM even + * though it is numerically greater, and all other stakinds appear in + * numerical order. We duplicate this quirk for consistency. + */ + if (do_bounds_histogram) + { + bool converted = false; + Datum stavalues; + + stavalues = text_to_stavalues("range_bounds_histogram", + &array_in_fn, + PG_GETARG_DATUM(RANGE_BOUNDS_HISTOGRAM_ARG), + atttypid, atttypmod, + elevel, &converted); + + if (converted) + { + set_stats_slot(values, nulls, replaces, + STATISTIC_KIND_BOUNDS_HISTOGRAM, + InvalidOid, InvalidOid, + 0, true, stavalues, false); + } + else + result = false; + } + + /* STATISTIC_KIND_RANGE_LENGTH_HISTOGRAM */ + if (do_range_length_histogram) + { + /* The anyarray is always a float8[] for this stakind */ + Datum elems[] = {PG_GETARG_DATUM(RANGE_EMPTY_FRAC_ARG)}; + ArrayType *arry = construct_array_builtin(elems, 1, FLOAT4OID); + Datum stanumbers = PointerGetDatum(arry); + + bool converted = false; + Datum stavalues; + + stavalues = text_to_stavalues("range_length_histogram", + &array_in_fn, + PG_GETARG_DATUM(RANGE_LENGTH_HISTOGRAM_ARG), + FLOAT8OID, 0, elevel, &converted); + + if (converted) + { + set_stats_slot(values, nulls, replaces, + STATISTIC_KIND_RANGE_LENGTH_HISTOGRAM, + Float8LessOperator, InvalidOid, + stanumbers, false, stavalues, false); + } + else + result = false; + } + + upsert_pg_statistic(starel, statup, values, nulls, replaces); + + if (HeapTupleIsValid(statup)) + ReleaseSysCache(statup); + table_close(starel, RowExclusiveLock); + + return result; +} + +/* + * If this relation is an index and that index has expressions in it, and + * the attnum specified is known to be an expression, then we must walk + * the list attributes up to the specified attnum to get the right + * expression. + */ +static Node * +get_attr_expr(Relation rel, int attnum) +{ + if ((rel->rd_rel->relkind == RELKIND_INDEX + || (rel->rd_rel->relkind == RELKIND_PARTITIONED_INDEX)) + && (rel->rd_indexprs != NIL) + && (rel->rd_index->indkey.values[attnum - 1] == 0)) + { + ListCell *indexpr_item = list_head(rel->rd_indexprs); + + for (int i = 0; i < attnum - 1; i++) + if (rel->rd_index->indkey.values[i] == 0) + indexpr_item = lnext(rel->rd_indexprs, indexpr_item); + + if (indexpr_item == NULL) /* shouldn't happen */ + elog(ERROR, "too few entries in indexprs list"); + + return (Node *) lfirst(indexpr_item); + } + return NULL; +} + +/* + * Derive type information from the attribute. + */ +static void +get_attr_stat_type(Oid reloid, AttrNumber attnum, int elevel, + Oid *atttypid, int32 *atttypmod, + char *atttyptype, Oid *atttypcoll, + Oid *eq_opr, Oid *lt_opr) +{ + Relation rel = relation_open(reloid, AccessShareLock); + Form_pg_attribute attr; + HeapTuple atup; + Node *expr; + TypeCacheEntry *typcache; + + atup = SearchSysCache2(ATTNUM, ObjectIdGetDatum(reloid), + Int16GetDatum(attnum)); + + /* Attribute not found */ + if (!HeapTupleIsValid(atup)) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_COLUMN), + errmsg("attribute %d of relation \"%s\" does not exist", + attnum, RelationGetRelationName(rel)))); + + attr = (Form_pg_attribute) GETSTRUCT(atup); + + if (attr->attisdropped) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_COLUMN), + errmsg("attribute %d of relation \"%s\" does not exist", + attnum, RelationGetRelationName(rel)))); + + expr = get_attr_expr(rel, attr->attnum); + + /* + * When analyzing an expression index, believe the expression tree's type + * not the column datatype --- the latter might be the opckeytype storage + * type of the opclass, which is not interesting for our purposes. This + * mimics the behvior of examine_attribute(). + */ + if (expr == NULL) + { + *atttypid = attr->atttypid; + *atttypmod = attr->atttypmod; + *atttypcoll = attr->attcollation; + } + else + { + *atttypid = exprType(expr); + *atttypmod = exprTypmod(expr); + + if (OidIsValid(attr->attcollation)) + *atttypcoll = attr->attcollation; + else + *atttypcoll = exprCollation(expr); + } + ReleaseSysCache(atup); + + /* + * If it's a multirange, step down to the range type, as is done by + * multirange_typanalyze(). + */ + if (type_is_multirange(*atttypid)) + *atttypid = get_multirange_range(*atttypid); + + /* finds the right operators even if atttypid is a domain */ + typcache = lookup_type_cache(*atttypid, TYPECACHE_LT_OPR | TYPECACHE_EQ_OPR); + *atttyptype = typcache->typtype; + *eq_opr = typcache->eq_opr; + *lt_opr = typcache->lt_opr; + + /* + * Special case: collation for tsvector is DEFAULT_COLLATION_OID. See + * compute_tsvector_stats(). + */ + if (*atttypid == TSVECTOROID) + *atttypcoll = DEFAULT_COLLATION_OID; + + relation_close(rel, NoLock); +} + +/* + * Derive element type information from the attribute type. + */ +static bool +get_elem_stat_type(Oid atttypid, char atttyptype, int elevel, + Oid *elemtypid, Oid *elem_eq_opr) +{ + TypeCacheEntry *elemtypcache; + + if (atttypid == TSVECTOROID) + { + /* + * Special case: element type for tsvector is text. See + * compute_tsvector_stats(). + */ + *elemtypid = TEXTOID; + } + else + { + /* find underlying element type through any domain */ + *elemtypid = get_base_element_type(atttypid); + } + + if (!OidIsValid(*elemtypid)) + return false; + + /* finds the right operator even if elemtypid is a domain */ + elemtypcache = lookup_type_cache(*elemtypid, TYPECACHE_EQ_OPR); + if (!OidIsValid(elemtypcache->eq_opr)) + return false; + + *elem_eq_opr = elemtypcache->eq_opr; + + return true; +} + +/* + * Cast a text datum into an array with element type elemtypid. + * + * If an error is encountered, capture it and re-throw at elevel, and set ok + * to false. If the resulting array contains NULLs, raise an error at elevel + * and set ok to false. Otherwise, set ok to true. + */ +static Datum +text_to_stavalues(const char *staname, FmgrInfo *array_in, Datum d, Oid typid, + int32 typmod, int elevel, bool *ok) +{ + LOCAL_FCINFO(fcinfo, 8); + char *s; + Datum result; + ErrorSaveContext escontext = {T_ErrorSaveContext}; + + escontext.details_wanted = true; + + s = TextDatumGetCString(d); + + InitFunctionCallInfoData(*fcinfo, array_in, 3, InvalidOid, + (Node *) &escontext, NULL); + + fcinfo->args[0].value = CStringGetDatum(s); + fcinfo->args[0].isnull = false; + fcinfo->args[1].value = ObjectIdGetDatum(typid); + fcinfo->args[1].isnull = false; + fcinfo->args[2].value = Int32GetDatum(typmod); + fcinfo->args[2].isnull = false; + + result = FunctionCallInvoke(fcinfo); + + pfree(s); + + if (SOFT_ERROR_OCCURRED(&escontext)) + { + if (elevel != ERROR) + escontext.error_data->elevel = elevel; + ThrowErrorData(escontext.error_data); + *ok = false; + return (Datum) 0; + } + + if (array_contains_nulls(DatumGetArrayTypeP(result))) + { + ereport(elevel, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("\"%s\" array cannot contain NULL values", staname))); + *ok = false; + return (Datum) 0; + } + + *ok = true; + + return result; +} + +/* + * Find and update the slot with the given stakind, or use the first empty + * slot. + */ +static void +set_stats_slot(Datum *values, bool *nulls, bool *replaces, + int16 stakind, Oid staop, Oid stacoll, + Datum stanumbers, bool stanumbers_isnull, + Datum stavalues, bool stavalues_isnull) +{ + int slotidx; + int first_empty = -1; + AttrNumber stakind_attnum; + AttrNumber staop_attnum; + AttrNumber stacoll_attnum; + + /* find existing slot with given stakind */ + for (slotidx = 0; slotidx < STATISTIC_NUM_SLOTS; slotidx++) + { + stakind_attnum = Anum_pg_statistic_stakind1 - 1 + slotidx; + + if (first_empty < 0 && + DatumGetInt16(values[stakind_attnum]) == 0) + first_empty = slotidx; + if (DatumGetInt16(values[stakind_attnum]) == stakind) + break; + } + + if (slotidx >= STATISTIC_NUM_SLOTS && first_empty >= 0) + slotidx = first_empty; + + if (slotidx >= STATISTIC_NUM_SLOTS) + ereport(ERROR, + (errmsg("maximum number of statistics slots exceeded: %d", + slotidx + 1))); + + stakind_attnum = Anum_pg_statistic_stakind1 - 1 + slotidx; + staop_attnum = Anum_pg_statistic_staop1 - 1 + slotidx; + stacoll_attnum = Anum_pg_statistic_stacoll1 - 1 + slotidx; + + if (DatumGetInt16(values[stakind_attnum]) != stakind) + { + values[stakind_attnum] = Int16GetDatum(stakind); + replaces[stakind_attnum] = true; + } + if (DatumGetObjectId(values[staop_attnum]) != staop) + { + values[staop_attnum] = ObjectIdGetDatum(staop); + replaces[staop_attnum] = true; + } + if (DatumGetObjectId(values[stacoll_attnum]) != stacoll) + { + values[stacoll_attnum] = ObjectIdGetDatum(stacoll); + replaces[stacoll_attnum] = true; + } + if (!stanumbers_isnull) + { + values[Anum_pg_statistic_stanumbers1 - 1 + slotidx] = stanumbers; + nulls[Anum_pg_statistic_stanumbers1 - 1 + slotidx] = false; + replaces[Anum_pg_statistic_stanumbers1 - 1 + slotidx] = true; + } + if (!stavalues_isnull) + { + values[Anum_pg_statistic_stavalues1 - 1 + slotidx] = stavalues; + nulls[Anum_pg_statistic_stavalues1 - 1 + slotidx] = false; + replaces[Anum_pg_statistic_stavalues1 - 1 + slotidx] = true; + } +} + +/* + * Upsert the pg_statistic record. + */ +static void +upsert_pg_statistic(Relation starel, HeapTuple oldtup, + Datum *values, bool *nulls, bool *replaces) +{ + HeapTuple newtup; + + if (HeapTupleIsValid(oldtup)) + { + newtup = heap_modify_tuple(oldtup, RelationGetDescr(starel), + values, nulls, replaces); + CatalogTupleUpdate(starel, &newtup->t_self, newtup); + } + else + { + newtup = heap_form_tuple(RelationGetDescr(starel), values, nulls); + CatalogTupleInsert(starel, newtup); + } + + heap_freetuple(newtup); +} + +/* + * Delete pg_statistic record. + */ +static bool +delete_pg_statistic(Oid reloid, AttrNumber attnum, bool stainherit) +{ + Relation sd = table_open(StatisticRelationId, RowExclusiveLock); + HeapTuple oldtup; + + /* Is there already a pg_statistic tuple for this attribute? */ + oldtup = SearchSysCache3(STATRELATTINH, + ObjectIdGetDatum(reloid), + Int16GetDatum(attnum), + BoolGetDatum(stainherit)); + + if (HeapTupleIsValid(oldtup)) + { + CatalogTupleDelete(sd, &oldtup->t_self); + ReleaseSysCache(oldtup); + table_close(sd, RowExclusiveLock); + return true; + } + + table_close(sd, RowExclusiveLock); + return false; +} + +/* + * Initialize values and nulls for a new stats tuple. + */ +static void +init_empty_stats_tuple(Oid reloid, int16 attnum, bool inherited, + Datum *values, bool *nulls, bool *replaces) +{ + memset(nulls, true, sizeof(bool) * Natts_pg_statistic); + memset(replaces, true, sizeof(bool) * Natts_pg_statistic); + + /* must initialize non-NULL attributes */ + + values[Anum_pg_statistic_starelid - 1] = ObjectIdGetDatum(reloid); + nulls[Anum_pg_statistic_starelid - 1] = false; + values[Anum_pg_statistic_staattnum - 1] = Int16GetDatum(attnum); + nulls[Anum_pg_statistic_staattnum - 1] = false; + values[Anum_pg_statistic_stainherit - 1] = BoolGetDatum(inherited); + nulls[Anum_pg_statistic_stainherit - 1] = false; + + values[Anum_pg_statistic_stanullfrac - 1] = DEFAULT_NULL_FRAC; + nulls[Anum_pg_statistic_stanullfrac - 1] = false; + values[Anum_pg_statistic_stawidth - 1] = DEFAULT_AVG_WIDTH; + nulls[Anum_pg_statistic_stawidth - 1] = false; + values[Anum_pg_statistic_stadistinct - 1] = DEFAULT_N_DISTINCT; + nulls[Anum_pg_statistic_stadistinct - 1] = false; + + /* initialize stakind, staop, and stacoll slots */ + for (int slotnum = 0; slotnum < STATISTIC_NUM_SLOTS; slotnum++) + { + values[Anum_pg_statistic_stakind1 + slotnum - 1] = (Datum) 0; + nulls[Anum_pg_statistic_stakind1 + slotnum - 1] = false; + values[Anum_pg_statistic_staop1 + slotnum - 1] = InvalidOid; + nulls[Anum_pg_statistic_staop1 + slotnum - 1] = false; + values[Anum_pg_statistic_stacoll1 + slotnum - 1] = InvalidOid; + nulls[Anum_pg_statistic_stacoll1 + slotnum - 1] = false; + } +} + +/* + * Import statistics for a given relation attribute. + * + * Inserts or replaces a row in pg_statistic for the given relation and + * attribute name. It takes input parameters that correspond to columns in the + * view pg_stats. + * + * Parameters null_frac, avg_width, and n_distinct all correspond to NOT NULL + * columns in pg_statistic. The remaining parameters all belong to a specific + * stakind. Some stakinds require multiple parameters, which must be specified + * together (or neither specified). + * + * Parameters are only superficially validated. Omitting a parameter or + * passing NULL leaves the statistic unchanged. + * + * Parameters corresponding to ANYARRAY columns are instead passed in as text + * values, which is a valid input string for an array of the type or element + * type of the attribute. Any error generated by the array_in() function will + * in turn fail the function. + */ +Datum +pg_set_attribute_stats(PG_FUNCTION_ARGS) +{ + attribute_statistics_update(fcinfo, ERROR); + PG_RETURN_VOID(); +} + +/* + * Delete statistics for the given attribute. + */ +Datum +pg_clear_attribute_stats(PG_FUNCTION_ARGS) +{ + Oid reloid; + Name attname; + AttrNumber attnum; + bool inherited; + + stats_check_required_arg(fcinfo, attarginfo, ATTRELATION_ARG); + reloid = PG_GETARG_OID(ATTRELATION_ARG); + + stats_lock_check_privileges(reloid); + + stats_check_required_arg(fcinfo, attarginfo, ATTNAME_ARG); + attname = PG_GETARG_NAME(ATTNAME_ARG); + attnum = get_attnum(reloid, NameStr(*attname)); + + stats_check_required_arg(fcinfo, attarginfo, INHERITED_ARG); + inherited = PG_GETARG_BOOL(INHERITED_ARG); + + delete_pg_statistic(reloid, attnum, inherited); + PG_RETURN_VOID(); +} diff --git a/src/backend/statistics/meson.build b/src/backend/statistics/meson.build index 23648b3775f..77b8e259a74 100644 --- a/src/backend/statistics/meson.build +++ b/src/backend/statistics/meson.build @@ -1,6 +1,7 @@ # Copyright (c) 2022-2024, PostgreSQL Global Development Group backend_sources += files( + 'attribute_stats.c', 'dependencies.c', 'extended_stats.c', 'mcv.c', diff --git a/src/backend/statistics/stat_utils.c b/src/backend/statistics/stat_utils.c index 4babed2e5d7..e7997ec486e 100644 --- a/src/backend/statistics/stat_utils.c +++ b/src/backend/statistics/stat_utils.c @@ -40,6 +40,79 @@ stats_check_required_arg(FunctionCallInfo fcinfo, arginfo[argnum].argname))); } +/* + * Check that argument is either NULL or a one dimensional array with no + * NULLs. + * + * If a problem is found, emit at elevel, and return false. Otherwise return + * true. + */ +bool +stats_check_arg_array(FunctionCallInfo fcinfo, + struct StatsArgInfo *arginfo, + int argnum, int elevel) +{ + ArrayType *arr; + + if (PG_ARGISNULL(argnum)) + return true; + + arr = DatumGetArrayTypeP(PG_GETARG_DATUM(argnum)); + + if (ARR_NDIM(arr) != 1) + { + ereport(elevel, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("\"%s\" cannot be a multidimensional array", + arginfo[argnum].argname))); + return false; + } + + if (array_contains_nulls(arr)) + { + ereport(elevel, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("\"%s\" array cannot contain NULL values", + arginfo[argnum].argname))); + return false; + } + + return true; +} + +/* + * Enforce parameter pairs that must be specified together (or not at all) for + * a particular stakind, such as most_common_vals and most_common_freqs for + * STATISTIC_KIND_MCV. + * + * If a problem is found, emit at elevel, and return false. Otherwise return + * true. + */ +bool +stats_check_arg_pair(FunctionCallInfo fcinfo, + struct StatsArgInfo *arginfo, + int argnum1, int argnum2, int elevel) +{ + if (PG_ARGISNULL(argnum1) && PG_ARGISNULL(argnum2)) + return true; + + if (PG_ARGISNULL(argnum1) || PG_ARGISNULL(argnum2)) + { + int nullarg = PG_ARGISNULL(argnum1) ? argnum1 : argnum2; + int otherarg = PG_ARGISNULL(argnum1) ? argnum2 : argnum1; + + ereport(elevel, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("\"%s\" must be specified when \"%s\" is specified", + arginfo[nullarg].argname, + arginfo[otherarg].argname))); + + return false; + } + + return true; +} + /* * Lock relation in ShareUpdateExclusive mode, check privileges, and close the * relation (but retain the lock). diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 76e4634ea4e..429c195e652 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -12185,6 +12185,20 @@ proargnames => '{summarized_tli,summarized_lsn,pending_lsn,summarizer_pid}', prosrc => 'pg_get_wal_summarizer_state' }, # Statistics Import +{ oid => '9162', + descr => 'set statistics on attribute', + proname => 'pg_set_attribute_stats', provolatile => 'v', proisstrict => 'f', + proparallel => 'u', prorettype => 'void', + proargtypes => 'regclass name bool float4 int4 float4 text _float4 text float4 text _float4 _float4 text float4 text', + proargnames => '{relation,attname,inherited,null_frac,avg_width,n_distinct,most_common_vals,most_common_freqs,histogram_bounds,correlation,most_common_elems,most_common_elem_freqs,elem_count_histogram,range_length_histogram,range_empty_frac,range_bounds_histogram}', + prosrc => 'pg_set_attribute_stats' }, +{ oid => '9163', + descr => 'clear statistics on attribute', + proname => 'pg_clear_attribute_stats', provolatile => 'v', proisstrict => 'f', + proparallel => 'u', prorettype => 'void', + proargtypes => 'regclass name bool', + proargnames => '{relation,attname,inherited}', + prosrc => 'pg_clear_attribute_stats' }, { oid => '9944', descr => 'set statistics on relation', proname => 'pg_set_relation_stats', provolatile => 'v', proisstrict => 'f', diff --git a/src/include/statistics/stat_utils.h b/src/include/statistics/stat_utils.h index 46057c60c03..e5f19d00609 100644 --- a/src/include/statistics/stat_utils.h +++ b/src/include/statistics/stat_utils.h @@ -24,6 +24,13 @@ struct StatsArgInfo extern void stats_check_required_arg(FunctionCallInfo fcinfo, struct StatsArgInfo *arginfo, int argnum); +extern bool stats_check_arg_array(FunctionCallInfo fcinfo, + struct StatsArgInfo *arginfo, int argnum, + int elevel); +extern bool stats_check_arg_pair(FunctionCallInfo fcinfo, + struct StatsArgInfo *arginfo, + int argnum1, int argnum2, int elevel); + extern void stats_lock_check_privileges(Oid reloid); #endif /* STATS_UTILS_H */ diff --git a/src/test/regress/expected/stats_import.out b/src/test/regress/expected/stats_import.out index df39ac2b635..2868cc47f1d 100644 --- a/src/test/regress/expected/stats_import.out +++ b/src/test/regress/expected/stats_import.out @@ -158,7 +158,7 @@ SELECT relpages => 2::integer); pg_set_relation_stats ----------------------- - t + (1 row) -- nothing stops us from setting it to -1 @@ -168,13 +168,670 @@ SELECT relpages => -1::integer); pg_set_relation_stats ----------------------- - t + +(1 row) + +-- error: object doesn't exist +SELECT pg_catalog.pg_set_attribute_stats( + relation => '0'::oid, + attname => 'id'::name, + inherited => false::boolean, + null_frac => 0.1::real, + avg_width => 2::integer, + n_distinct => 0.3::real); +ERROR: could not open relation with OID 0 +-- error: relation null +SELECT pg_catalog.pg_set_attribute_stats( + relation => NULL::oid, + attname => 'id'::name, + inherited => false::boolean, + null_frac => 0.1::real, + avg_width => 2::integer, + n_distinct => 0.3::real); +ERROR: "relation" cannot be NULL +-- error: attname null +SELECT pg_catalog.pg_set_attribute_stats( + relation => 'stats_import.test'::regclass, + attname => NULL::name, + inherited => false::boolean, + null_frac => 0.1::real, + avg_width => 2::integer, + n_distinct => 0.3::real); +ERROR: "attname" cannot be NULL +-- error: inherited null +SELECT pg_catalog.pg_set_attribute_stats( + relation => 'stats_import.test'::regclass, + attname => 'id'::name, + inherited => NULL::boolean, + null_frac => 0.1::real, + avg_width => 2::integer, + n_distinct => 0.3::real); +ERROR: "inherited" cannot be NULL +-- error: null_frac null +SELECT pg_catalog.pg_set_attribute_stats( + relation => 'stats_import.test'::regclass, + attname => 'id'::name, + inherited => false::boolean, + null_frac => NULL::real, + avg_width => 2::integer, + n_distinct => 0.3::real); + pg_set_attribute_stats +------------------------ + +(1 row) + +-- error: avg_width null +SELECT pg_catalog.pg_set_attribute_stats( + relation => 'stats_import.test'::regclass, + attname => 'id'::name, + inherited => false::boolean, + null_frac => 0.1::real, + avg_width => NULL::integer, + n_distinct => 0.3::real); + pg_set_attribute_stats +------------------------ + +(1 row) + +-- error: avg_width null +SELECT pg_catalog.pg_set_attribute_stats( + relation => 'stats_import.test'::regclass, + attname => 'id'::name, + inherited => false::boolean, + null_frac => 0.1::real, + avg_width => 2::integer, + n_distinct => NULL::real); + pg_set_attribute_stats +------------------------ + +(1 row) + +-- ok: no stakinds +SELECT pg_catalog.pg_set_attribute_stats( + relation => 'stats_import.test'::regclass, + attname => 'id'::name, + inherited => false::boolean, + null_frac => 0.1::real, + avg_width => 2::integer, + n_distinct => 0.3::real); + pg_set_attribute_stats +------------------------ + +(1 row) + +SELECT stanullfrac, stawidth, stadistinct +FROM pg_statistic +WHERE starelid = 'stats_import.test'::regclass; + stanullfrac | stawidth | stadistinct +-------------+----------+------------- + 0.1 | 2 | 0.3 +(1 row) + +-- error: mcv / mcf null mismatch +SELECT pg_catalog.pg_set_attribute_stats( + relation => 'stats_import.test'::regclass, + attname => 'id'::name, + inherited => false::boolean, + null_frac => 0.5::real, + avg_width => 2::integer, + n_distinct => -0.1::real, + most_common_freqs => '{0.1,0.2,0.3}'::real[] + ); +ERROR: "most_common_vals" must be specified when "most_common_freqs" is specified +-- error: mcv / mcf null mismatch part 2 +SELECT pg_catalog.pg_set_attribute_stats( + relation => 'stats_import.test'::regclass, + attname => 'id'::name, + inherited => false::boolean, + null_frac => 0.5::real, + avg_width => 2::integer, + n_distinct => -0.1::real, + most_common_vals => '{1,2,3}'::text + ); +ERROR: "most_common_freqs" must be specified when "most_common_vals" is specified +-- error: mcv / mcf type mismatch +SELECT pg_catalog.pg_set_attribute_stats( + relation => 'stats_import.test'::regclass, + attname => 'id'::name, + inherited => false::boolean, + null_frac => 0.5::real, + avg_width => 2::integer, + n_distinct => -0.1::real, + most_common_vals => '{2023-09-30,2024-10-31,3}'::text, + most_common_freqs => '{0.2,0.1}'::real[] + ); +ERROR: invalid input syntax for type integer: "2023-09-30" +-- warning: mcv cast failure +SELECT pg_catalog.pg_set_attribute_stats( + relation => 'stats_import.test'::regclass, + attname => 'id'::name, + inherited => false::boolean, + null_frac => 0.5::real, + avg_width => 2::integer, + n_distinct => -0.1::real, + most_common_vals => '{2,four,3}'::text, + most_common_freqs => '{0.3,0.25,0.05}'::real[] + ); +ERROR: invalid input syntax for type integer: "four" +-- ok: mcv+mcf +SELECT pg_catalog.pg_set_attribute_stats( + relation => 'stats_import.test'::regclass, + attname => 'id'::name, + inherited => false::boolean, + null_frac => 0.5::real, + avg_width => 2::integer, + n_distinct => -0.1::real, + most_common_vals => '{2,1,3}'::text, + most_common_freqs => '{0.3,0.25,0.05}'::real[] + ); + pg_set_attribute_stats +------------------------ + +(1 row) + +SELECT * +FROM pg_stats +WHERE schemaname = 'stats_import' +AND tablename = 'test' +AND inherited = false +AND attname = 'id'; + schemaname | tablename | attname | inherited | null_frac | avg_width | n_distinct | most_common_vals | most_common_freqs | histogram_bounds | correlation | most_common_elems | most_common_elem_freqs | elem_count_histogram | range_length_histogram | range_empty_frac | range_bounds_histogram +--------------+-----------+---------+-----------+-----------+-----------+------------+------------------+-------------------+------------------+-------------+-------------------+------------------------+----------------------+------------------------+------------------+------------------------ + stats_import | test | id | f | 0.5 | 2 | -0.1 | {2,1,3} | {0.3,0.25,0.05} | | | | | | | | +(1 row) + +-- error: histogram elements null value +-- this generates no warnings, but perhaps it should +SELECT pg_catalog.pg_set_attribute_stats( + relation => 'stats_import.test'::regclass, + attname => 'id'::name, + inherited => false::boolean, + null_frac => 0.5::real, + avg_width => 2::integer, + n_distinct => -0.1::real, + histogram_bounds => '{1,NULL,3,4}'::text + ); +ERROR: "histogram_bounds" array cannot contain NULL values +-- ok: histogram_bounds +SELECT pg_catalog.pg_set_attribute_stats( + relation => 'stats_import.test'::regclass, + attname => 'id'::name, + inherited => false::boolean, + null_frac => 0.5::real, + avg_width => 2::integer, + n_distinct => -0.1::real, + histogram_bounds => '{1,2,3,4}'::text + ); + pg_set_attribute_stats +------------------------ + +(1 row) + +SELECT * +FROM pg_stats +WHERE schemaname = 'stats_import' +AND tablename = 'test' +AND inherited = false +AND attname = 'id'; + schemaname | tablename | attname | inherited | null_frac | avg_width | n_distinct | most_common_vals | most_common_freqs | histogram_bounds | correlation | most_common_elems | most_common_elem_freqs | elem_count_histogram | range_length_histogram | range_empty_frac | range_bounds_histogram +--------------+-----------+---------+-----------+-----------+-----------+------------+------------------+-------------------+------------------+-------------+-------------------+------------------------+----------------------+------------------------+------------------+------------------------ + stats_import | test | id | f | 0.5 | 2 | -0.1 | {2,1,3} | {0.3,0.25,0.05} | {1,2,3,4} | | | | | | | +(1 row) + +-- ok: correlation +SELECT pg_catalog.pg_set_attribute_stats( + relation => 'stats_import.test'::regclass, + attname => 'id'::name, + inherited => false::boolean, + null_frac => 0.5::real, + avg_width => 2::integer, + n_distinct => -0.1::real, + correlation => 0.5::real); + pg_set_attribute_stats +------------------------ + +(1 row) + +SELECT * +FROM pg_stats +WHERE schemaname = 'stats_import' +AND tablename = 'test' +AND inherited = false +AND attname = 'id'; + schemaname | tablename | attname | inherited | null_frac | avg_width | n_distinct | most_common_vals | most_common_freqs | histogram_bounds | correlation | most_common_elems | most_common_elem_freqs | elem_count_histogram | range_length_histogram | range_empty_frac | range_bounds_histogram +--------------+-----------+---------+-----------+-----------+-----------+------------+------------------+-------------------+------------------+-------------+-------------------+------------------------+----------------------+------------------------+------------------+------------------------ + stats_import | test | id | f | 0.5 | 2 | -0.1 | {2,1,3} | {0.3,0.25,0.05} | {1,2,3,4} | 0.5 | | | | | | +(1 row) + +-- error: scalars can't have mcelem +SELECT pg_catalog.pg_set_attribute_stats( + relation => 'stats_import.test'::regclass, + attname => 'id'::name, + inherited => false::boolean, + null_frac => 0.5::real, + avg_width => 2::integer, + n_distinct => -0.1::real, + most_common_elems => '{1,3}'::text, + most_common_elem_freqs => '{0.3,0.2,0.2,0.3,0.0}'::real[] + ); +ERROR: unable to determine element type of attribute "id" +DETAIL: Cannot set STATISTIC_KIND_MCELEM or STATISTIC_KIND_DECHIST. +-- error: mcelem / mcelem mismatch +SELECT pg_catalog.pg_set_attribute_stats( + relation => 'stats_import.test'::regclass, + attname => 'tags'::name, + inherited => false::boolean, + null_frac => 0.5::real, + avg_width => 2::integer, + n_distinct => -0.1::real, + most_common_elems => '{one,two}'::text + ); +ERROR: "most_common_elem_freqs" must be specified when "most_common_elems" is specified +-- error: mcelem / mcelem null mismatch part 2 +SELECT pg_catalog.pg_set_attribute_stats( + relation => 'stats_import.test'::regclass, + attname => 'tags'::name, + inherited => false::boolean, + null_frac => 0.5::real, + avg_width => 2::integer, + n_distinct => -0.1::real, + most_common_elem_freqs => '{0.3,0.2,0.2,0.3}'::real[] + ); +ERROR: "most_common_elems" must be specified when "most_common_elem_freqs" is specified +-- ok: mcelem +SELECT pg_catalog.pg_set_attribute_stats( + relation => 'stats_import.test'::regclass, + attname => 'tags'::name, + inherited => false::boolean, + null_frac => 0.5::real, + avg_width => 2::integer, + n_distinct => -0.1::real, + most_common_elems => '{one,three}'::text, + most_common_elem_freqs => '{0.3,0.2,0.2,0.3,0.0}'::real[] + ); + pg_set_attribute_stats +------------------------ + +(1 row) + +SELECT * +FROM pg_stats +WHERE schemaname = 'stats_import' +AND tablename = 'test' +AND inherited = false +AND attname = 'tags'; + schemaname | tablename | attname | inherited | null_frac | avg_width | n_distinct | most_common_vals | most_common_freqs | histogram_bounds | correlation | most_common_elems | most_common_elem_freqs | elem_count_histogram | range_length_histogram | range_empty_frac | range_bounds_histogram +--------------+-----------+---------+-----------+-----------+-----------+------------+------------------+-------------------+------------------+-------------+-------------------+------------------------+----------------------+------------------------+------------------+------------------------ + stats_import | test | tags | f | 0.5 | 2 | -0.1 | | | | | {one,three} | {0.3,0.2,0.2,0.3,0} | | | | +(1 row) + +-- error: scalars can't have elem_count_histogram +SELECT pg_catalog.pg_set_attribute_stats( + relation => 'stats_import.test'::regclass, + attname => 'id'::name, + inherited => false::boolean, + null_frac => 0.5::real, + avg_width => 2::integer, + n_distinct => -0.1::real, + elem_count_histogram => '{1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}'::real[] + ); +ERROR: unable to determine element type of attribute "id" +DETAIL: Cannot set STATISTIC_KIND_MCELEM or STATISTIC_KIND_DECHIST. +-- error: elem_count_histogram null element +SELECT pg_catalog.pg_set_attribute_stats( + relation => 'stats_import.test'::regclass, + attname => 'tags'::name, + inherited => false::boolean, + null_frac => 0.5::real, + avg_width => 2::integer, + n_distinct => -0.1::real, + elem_count_histogram => '{1,1,NULL,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}'::real[] + ); +ERROR: "elem_count_histogram" array cannot contain NULL values +-- ok: elem_count_histogram +SELECT pg_catalog.pg_set_attribute_stats( + relation => 'stats_import.test'::regclass, + attname => 'tags'::name, + inherited => false::boolean, + null_frac => 0.5::real, + avg_width => 2::integer, + n_distinct => -0.1::real, + elem_count_histogram => '{1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}'::real[] + ); + pg_set_attribute_stats +------------------------ + +(1 row) + +SELECT * +FROM pg_stats +WHERE schemaname = 'stats_import' +AND tablename = 'test' +AND inherited = false +AND attname = 'tags'; + schemaname | tablename | attname | inherited | null_frac | avg_width | n_distinct | most_common_vals | most_common_freqs | histogram_bounds | correlation | most_common_elems | most_common_elem_freqs | elem_count_histogram | range_length_histogram | range_empty_frac | range_bounds_histogram +--------------+-----------+---------+-----------+-----------+-----------+------------+------------------+-------------------+------------------+-------------+-------------------+------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------+------------------+------------------------ + stats_import | test | tags | f | 0.5 | 2 | -0.1 | | | | | {one,three} | {0.3,0.2,0.2,0.3,0} | {1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1} | | | +(1 row) + +-- error: scalars can't have range stats +SELECT pg_catalog.pg_set_attribute_stats( + relation => 'stats_import.test'::regclass, + attname => 'id'::name, + inherited => false::boolean, + null_frac => 0.5::real, + avg_width => 2::integer, + n_distinct => -0.1::real, + range_empty_frac => 0.5::real, + range_length_histogram => '{399,499,Infinity}'::text + ); +ERROR: attribute "id" is not a range type +DETAIL: Cannot set STATISTIC_KIND_RANGE_LENGTH_HISTOGRAM or STATISTIC_KIND_BOUNDS_HISTOGRAM. +-- error: range_empty_frac range_length_hist null mismatch +SELECT pg_catalog.pg_set_attribute_stats( + relation => 'stats_import.test'::regclass, + attname => 'arange'::name, + inherited => false::boolean, + null_frac => 0.5::real, + avg_width => 2::integer, + n_distinct => -0.1::real, + range_length_histogram => '{399,499,Infinity}'::text + ); +ERROR: "range_empty_frac" must be specified when "range_length_histogram" is specified +-- error: range_empty_frac range_length_hist null mismatch part 2 +SELECT pg_catalog.pg_set_attribute_stats( + relation => 'stats_import.test'::regclass, + attname => 'arange'::name, + inherited => false::boolean, + null_frac => 0.5::real, + avg_width => 2::integer, + n_distinct => -0.1::real, + range_empty_frac => 0.5::real + ); +ERROR: "range_length_histogram" must be specified when "range_empty_frac" is specified +-- ok: range_empty_frac + range_length_hist +SELECT pg_catalog.pg_set_attribute_stats( + relation => 'stats_import.test'::regclass, + attname => 'arange'::name, + inherited => false::boolean, + null_frac => 0.5::real, + avg_width => 2::integer, + n_distinct => -0.1::real, + range_empty_frac => 0.5::real, + range_length_histogram => '{399,499,Infinity}'::text + ); + pg_set_attribute_stats +------------------------ + +(1 row) + +SELECT * +FROM pg_stats +WHERE schemaname = 'stats_import' +AND tablename = 'test' +AND inherited = false +AND attname = 'arange'; + schemaname | tablename | attname | inherited | null_frac | avg_width | n_distinct | most_common_vals | most_common_freqs | histogram_bounds | correlation | most_common_elems | most_common_elem_freqs | elem_count_histogram | range_length_histogram | range_empty_frac | range_bounds_histogram +--------------+-----------+---------+-----------+-----------+-----------+------------+------------------+-------------------+------------------+-------------+-------------------+------------------------+----------------------+------------------------+------------------+------------------------ + stats_import | test | arange | f | 0.5 | 2 | -0.1 | | | | | | | | {399,499,Infinity} | 0.5 | +(1 row) + +-- error: scalars can't have range stats +SELECT pg_catalog.pg_set_attribute_stats( + relation => 'stats_import.test'::regclass, + attname => 'id'::name, + inherited => false::boolean, + null_frac => 0.5::real, + avg_width => 2::integer, + n_distinct => -0.1::real, + range_bounds_histogram => '{"[-1,1)","[0,4)","[1,4)","[1,100)"}'::text + ); +ERROR: attribute "id" is not a range type +DETAIL: Cannot set STATISTIC_KIND_RANGE_LENGTH_HISTOGRAM or STATISTIC_KIND_BOUNDS_HISTOGRAM. +-- ok: range_bounds_histogram +SELECT pg_catalog.pg_set_attribute_stats( + relation => 'stats_import.test'::regclass, + attname => 'arange'::name, + inherited => false::boolean, + null_frac => 0.5::real, + avg_width => 2::integer, + n_distinct => -0.1::real, + range_bounds_histogram => '{"[-1,1)","[0,4)","[1,4)","[1,100)"}'::text + ); + pg_set_attribute_stats +------------------------ + (1 row) +SELECT * +FROM pg_stats +WHERE schemaname = 'stats_import' +AND tablename = 'test' +AND inherited = false +AND attname = 'arange'; + schemaname | tablename | attname | inherited | null_frac | avg_width | n_distinct | most_common_vals | most_common_freqs | histogram_bounds | correlation | most_common_elems | most_common_elem_freqs | elem_count_histogram | range_length_histogram | range_empty_frac | range_bounds_histogram +--------------+-----------+---------+-----------+-----------+-----------+------------+------------------+-------------------+------------------+-------------+-------------------+------------------------+----------------------+------------------------+------------------+-------------------------------------- + stats_import | test | arange | f | 0.5 | 2 | -0.1 | | | | | | | | {399,499,Infinity} | 0.5 | {"[-1,1)","[0,4)","[1,4)","[1,100)"} +(1 row) + +-- error: cannot set most_common_elems for range type +SELECT pg_catalog.pg_set_attribute_stats( + relation => 'stats_import.test'::regclass, + attname => 'arange'::name, + inherited => false::boolean, + null_frac => 0.5::real, + avg_width => 2::integer, + n_distinct => -0.1::real, + most_common_vals => '{"[2,3)","[1,2)","[3,4)"}'::text, + most_common_freqs => '{0.3,0.25,0.05}'::real[], + histogram_bounds => '{"[1,2)","[2,3)","[3,4)","[4,5)"}'::text, + correlation => 1.1::real, + most_common_elems => '{3,1}'::text, + most_common_elem_freqs => '{0.3,0.2,0.2,0.3,0.0}'::real[], + range_empty_frac => -0.5::real, + range_length_histogram => '{399,499,Infinity}'::text, + range_bounds_histogram => '{"[-1,1)","[0,4)","[1,4)","[1,100)"}'::text + ); +ERROR: unable to determine element type of attribute "arange" +DETAIL: Cannot set STATISTIC_KIND_MCELEM or STATISTIC_KIND_DECHIST. +-- +-- Test the ability to exactly copy data from one table to an identical table, +-- correctly reconstructing the stakind order as well as the staopN and +-- stacollN values. Because oids are not stable across databases, we can only +-- test this when the source and destination are on the same database +-- instance. For that reason, we borrow and adapt a query found in fe_utils +-- and used by pg_dump/pg_upgrade. +-- +INSERT INTO stats_import.test +SELECT 1, 'one', (1, 1.1, 'ONE', '2001-01-01', '{ "xkey": "xval" }')::stats_import.complex_type, int4range(1,4), array['red','green'] +UNION ALL +SELECT 2, 'two', (2, 2.2, 'TWO', '2002-02-02', '[true, 4, "six"]')::stats_import.complex_type, int4range(1,4), array['blue','yellow'] +UNION ALL +SELECT 3, 'tre', (3, 3.3, 'TRE', '2003-03-03', NULL)::stats_import.complex_type, int4range(-1,1), array['"orange"', 'purple', 'cyan'] +UNION ALL +SELECT 4, 'four', NULL, int4range(0,100), NULL; +CREATE INDEX is_odd ON stats_import.test(((comp).a % 2 = 1)); +-- Generate statistics on table with data +ANALYZE stats_import.test; +CREATE TABLE stats_import.test_clone ( LIKE stats_import.test ) + WITH (autovacuum_enabled = false); +CREATE INDEX is_odd_clone ON stats_import.test_clone(((comp).a % 2 = 1)); +-- +-- Copy stats from test to test_clone, and is_odd to is_odd_clone +-- +SELECT s.schemaname, s.tablename, s.attname, s.inherited +FROM pg_catalog.pg_stats AS s +CROSS JOIN LATERAL + pg_catalog.pg_set_attribute_stats( + relation => ('stats_import.' || s.tablename || '_clone')::regclass::oid, + attname => s.attname, + inherited => s.inherited, + null_frac => s.null_frac, + avg_width => s.avg_width, + n_distinct => s.n_distinct, + most_common_vals => s.most_common_vals::text, + most_common_freqs => s.most_common_freqs, + histogram_bounds => s.histogram_bounds::text, + correlation => s.correlation, + most_common_elems => s.most_common_elems::text, + most_common_elem_freqs => s.most_common_elem_freqs, + elem_count_histogram => s.elem_count_histogram, + range_bounds_histogram => s.range_bounds_histogram::text, + range_empty_frac => s.range_empty_frac, + range_length_histogram => s.range_length_histogram::text) AS r +WHERE s.schemaname = 'stats_import' +AND s.tablename IN ('test', 'is_odd') +ORDER BY s.tablename, s.attname, s.inherited; + schemaname | tablename | attname | inherited +--------------+-----------+---------+----------- + stats_import | is_odd | expr | f + stats_import | test | arange | f + stats_import | test | comp | f + stats_import | test | id | f + stats_import | test | name | f + stats_import | test | tags | f +(6 rows) + +SELECT c.relname, COUNT(*) AS num_stats +FROM pg_class AS c +JOIN pg_statistic s ON s.starelid = c.oid +WHERE c.relnamespace = 'stats_import'::regnamespace +AND c.relname IN ('test', 'test_clone', 'is_odd', 'is_odd_clone') +GROUP BY c.relname +ORDER BY c.relname; + relname | num_stats +--------------+----------- + is_odd | 1 + is_odd_clone | 1 + test | 5 + test_clone | 5 +(4 rows) + +-- check test minus test_clone +SELECT + a.attname, s.stainherit, s.stanullfrac, s.stawidth, s.stadistinct, + s.stakind1, s.stakind2, s.stakind3, s.stakind4, s.stakind5, + s.staop1, s.staop2, s.staop3, s.staop4, s.staop5, + s.stacoll1, s.stacoll2, s.stacoll3, s.stacoll4, s.stacoll5, + s.stanumbers1, s.stanumbers2, s.stanumbers3, s.stanumbers4, s.stanumbers5, + s.stavalues1::text AS sv1, s.stavalues2::text AS sv2, + s.stavalues3::text AS sv3, s.stavalues4::text AS sv4, + s.stavalues5::text AS sv5, 'test' AS direction +FROM pg_statistic s +JOIN pg_attribute a ON a.attrelid = s.starelid AND a.attnum = s.staattnum +WHERE s.starelid = 'stats_import.test'::regclass +EXCEPT +SELECT + a.attname, s.stainherit, s.stanullfrac, s.stawidth, s.stadistinct, + s.stakind1, s.stakind2, s.stakind3, s.stakind4, s.stakind5, + s.staop1, s.staop2, s.staop3, s.staop4, s.staop5, + s.stacoll1, s.stacoll2, s.stacoll3, s.stacoll4, s.stacoll5, + s.stanumbers1, s.stanumbers2, s.stanumbers3, s.stanumbers4, s.stanumbers5, + s.stavalues1::text AS sv1, s.stavalues2::text AS sv2, + s.stavalues3::text AS sv3, s.stavalues4::text AS sv4, + s.stavalues5::text AS sv5, 'test' AS direction +FROM pg_statistic s +JOIN pg_attribute a ON a.attrelid = s.starelid AND a.attnum = s.staattnum +WHERE s.starelid = 'stats_import.test_clone'::regclass; + attname | stainherit | stanullfrac | stawidth | stadistinct | stakind1 | stakind2 | stakind3 | stakind4 | stakind5 | staop1 | staop2 | staop3 | staop4 | staop5 | stacoll1 | stacoll2 | stacoll3 | stacoll4 | stacoll5 | stanumbers1 | stanumbers2 | stanumbers3 | stanumbers4 | stanumbers5 | sv1 | sv2 | sv3 | sv4 | sv5 | direction +---------+------------+-------------+----------+-------------+----------+----------+----------+----------+----------+--------+--------+--------+--------+--------+----------+----------+----------+----------+----------+-------------+-------------+-------------+-------------+-------------+-----+-----+-----+-----+-----+----------- +(0 rows) + +-- check test_clone minus test +SELECT + a.attname, s.stainherit, s.stanullfrac, s.stawidth, s.stadistinct, + s.stakind1, s.stakind2, s.stakind3, s.stakind4, s.stakind5, + s.staop1, s.staop2, s.staop3, s.staop4, s.staop5, + s.stacoll1, s.stacoll2, s.stacoll3, s.stacoll4, s.stacoll5, + s.stanumbers1, s.stanumbers2, s.stanumbers3, s.stanumbers4, s.stanumbers5, + s.stavalues1::text AS sv1, s.stavalues2::text AS sv2, + s.stavalues3::text AS sv3, s.stavalues4::text AS sv4, + s.stavalues5::text AS sv5, 'test_clone' AS direction +FROM pg_statistic s +JOIN pg_attribute a ON a.attrelid = s.starelid AND a.attnum = s.staattnum +WHERE s.starelid = 'stats_import.test_clone'::regclass +EXCEPT +SELECT + a.attname, s.stainherit, s.stanullfrac, s.stawidth, s.stadistinct, + s.stakind1, s.stakind2, s.stakind3, s.stakind4, s.stakind5, + s.staop1, s.staop2, s.staop3, s.staop4, s.staop5, + s.stacoll1, s.stacoll2, s.stacoll3, s.stacoll4, s.stacoll5, + s.stanumbers1, s.stanumbers2, s.stanumbers3, s.stanumbers4, s.stanumbers5, + s.stavalues1::text AS sv1, s.stavalues2::text AS sv2, + s.stavalues3::text AS sv3, s.stavalues4::text AS sv4, + s.stavalues5::text AS sv5, 'test_clone' AS direction +FROM pg_statistic s +JOIN pg_attribute a ON a.attrelid = s.starelid AND a.attnum = s.staattnum +WHERE s.starelid = 'stats_import.test'::regclass; + attname | stainherit | stanullfrac | stawidth | stadistinct | stakind1 | stakind2 | stakind3 | stakind4 | stakind5 | staop1 | staop2 | staop3 | staop4 | staop5 | stacoll1 | stacoll2 | stacoll3 | stacoll4 | stacoll5 | stanumbers1 | stanumbers2 | stanumbers3 | stanumbers4 | stanumbers5 | sv1 | sv2 | sv3 | sv4 | sv5 | direction +---------+------------+-------------+----------+-------------+----------+----------+----------+----------+----------+--------+--------+--------+--------+--------+----------+----------+----------+----------+----------+-------------+-------------+-------------+-------------+-------------+-----+-----+-----+-----+-----+----------- +(0 rows) + +-- check is_odd minus is_odd_clone +SELECT + a.attname, s.stainherit, s.stanullfrac, s.stawidth, s.stadistinct, + s.stakind1, s.stakind2, s.stakind3, s.stakind4, s.stakind5, + s.staop1, s.staop2, s.staop3, s.staop4, s.staop5, + s.stacoll1, s.stacoll2, s.stacoll3, s.stacoll4, s.stacoll5, + s.stanumbers1, s.stanumbers2, s.stanumbers3, s.stanumbers4, s.stanumbers5, + s.stavalues1::text AS sv1, s.stavalues2::text AS sv2, + s.stavalues3::text AS sv3, s.stavalues4::text AS sv4, + s.stavalues5::text AS sv5, 'is_odd' AS direction +FROM pg_statistic s +JOIN pg_attribute a ON a.attrelid = s.starelid AND a.attnum = s.staattnum +WHERE s.starelid = 'stats_import.is_odd'::regclass +EXCEPT +SELECT + a.attname, s.stainherit, s.stanullfrac, s.stawidth, s.stadistinct, + s.stakind1, s.stakind2, s.stakind3, s.stakind4, s.stakind5, + s.staop1, s.staop2, s.staop3, s.staop4, s.staop5, + s.stacoll1, s.stacoll2, s.stacoll3, s.stacoll4, s.stacoll5, + s.stanumbers1, s.stanumbers2, s.stanumbers3, s.stanumbers4, s.stanumbers5, + s.stavalues1::text AS sv1, s.stavalues2::text AS sv2, + s.stavalues3::text AS sv3, s.stavalues4::text AS sv4, + s.stavalues5::text AS sv5, 'is_odd' AS direction +FROM pg_statistic s +JOIN pg_attribute a ON a.attrelid = s.starelid AND a.attnum = s.staattnum +WHERE s.starelid = 'stats_import.is_odd_clone'::regclass; + attname | stainherit | stanullfrac | stawidth | stadistinct | stakind1 | stakind2 | stakind3 | stakind4 | stakind5 | staop1 | staop2 | staop3 | staop4 | staop5 | stacoll1 | stacoll2 | stacoll3 | stacoll4 | stacoll5 | stanumbers1 | stanumbers2 | stanumbers3 | stanumbers4 | stanumbers5 | sv1 | sv2 | sv3 | sv4 | sv5 | direction +---------+------------+-------------+----------+-------------+----------+----------+----------+----------+----------+--------+--------+--------+--------+--------+----------+----------+----------+----------+----------+-------------+-------------+-------------+-------------+-------------+-----+-----+-----+-----+-----+----------- +(0 rows) + +-- check is_odd_clone minus is_odd +SELECT + a.attname, s.stainherit, s.stanullfrac, s.stawidth, s.stadistinct, + s.stakind1, s.stakind2, s.stakind3, s.stakind4, s.stakind5, + s.staop1, s.staop2, s.staop3, s.staop4, s.staop5, + s.stacoll1, s.stacoll2, s.stacoll3, s.stacoll4, s.stacoll5, + s.stanumbers1, s.stanumbers2, s.stanumbers3, s.stanumbers4, s.stanumbers5, + s.stavalues1::text AS sv1, s.stavalues2::text AS sv2, + s.stavalues3::text AS sv3, s.stavalues4::text AS sv4, + s.stavalues5::text AS sv5, 'is_odd_clone' AS direction +FROM pg_statistic s +JOIN pg_attribute a ON a.attrelid = s.starelid AND a.attnum = s.staattnum +WHERE s.starelid = 'stats_import.is_odd_clone'::regclass +EXCEPT +SELECT + a.attname, s.stainherit, s.stanullfrac, s.stawidth, s.stadistinct, + s.stakind1, s.stakind2, s.stakind3, s.stakind4, s.stakind5, + s.staop1, s.staop2, s.staop3, s.staop4, s.staop5, + s.stacoll1, s.stacoll2, s.stacoll3, s.stacoll4, s.stacoll5, + s.stanumbers1, s.stanumbers2, s.stanumbers3, s.stanumbers4, s.stanumbers5, + s.stavalues1::text AS sv1, s.stavalues2::text AS sv2, + s.stavalues3::text AS sv3, s.stavalues4::text AS sv4, + s.stavalues5::text AS sv5, 'is_odd_clone' AS direction +FROM pg_statistic s +JOIN pg_attribute a ON a.attrelid = s.starelid AND a.attnum = s.staattnum +WHERE s.starelid = 'stats_import.is_odd'::regclass; + attname | stainherit | stanullfrac | stawidth | stadistinct | stakind1 | stakind2 | stakind3 | stakind4 | stakind5 | staop1 | staop2 | staop3 | staop4 | staop5 | stacoll1 | stacoll2 | stacoll3 | stacoll4 | stacoll5 | stanumbers1 | stanumbers2 | stanumbers3 | stanumbers4 | stanumbers5 | sv1 | sv2 | sv3 | sv4 | sv5 | direction +---------+------------+-------------+----------+-------------+----------+----------+----------+----------+----------+--------+--------+--------+--------+--------+----------+----------+----------+----------+----------+-------------+-------------+-------------+-------------+-------------+-----+-----+-----+-----+-----+----------- +(0 rows) + DROP SCHEMA stats_import CASCADE; -NOTICE: drop cascades to 5 other objects +NOTICE: drop cascades to 6 other objects DETAIL: drop cascades to type stats_import.complex_type drop cascades to table stats_import.test drop cascades to sequence stats_import.testseq drop cascades to view stats_import.testview drop cascades to table stats_import.part_parent +drop cascades to table stats_import.test_clone diff --git a/src/test/regress/sql/stats_import.sql b/src/test/regress/sql/stats_import.sql index 126b3ab9b9c..9b6c9094862 100644 --- a/src/test/regress/sql/stats_import.sql +++ b/src/test/regress/sql/stats_import.sql @@ -121,4 +121,549 @@ SELECT relation => 'stats_import.part_parent'::regclass, relpages => -1::integer); +-- error: object doesn't exist +SELECT pg_catalog.pg_set_attribute_stats( + relation => '0'::oid, + attname => 'id'::name, + inherited => false::boolean, + null_frac => 0.1::real, + avg_width => 2::integer, + n_distinct => 0.3::real); + +-- error: relation null +SELECT pg_catalog.pg_set_attribute_stats( + relation => NULL::oid, + attname => 'id'::name, + inherited => false::boolean, + null_frac => 0.1::real, + avg_width => 2::integer, + n_distinct => 0.3::real); + +-- error: attname null +SELECT pg_catalog.pg_set_attribute_stats( + relation => 'stats_import.test'::regclass, + attname => NULL::name, + inherited => false::boolean, + null_frac => 0.1::real, + avg_width => 2::integer, + n_distinct => 0.3::real); + +-- error: inherited null +SELECT pg_catalog.pg_set_attribute_stats( + relation => 'stats_import.test'::regclass, + attname => 'id'::name, + inherited => NULL::boolean, + null_frac => 0.1::real, + avg_width => 2::integer, + n_distinct => 0.3::real); + +-- error: null_frac null +SELECT pg_catalog.pg_set_attribute_stats( + relation => 'stats_import.test'::regclass, + attname => 'id'::name, + inherited => false::boolean, + null_frac => NULL::real, + avg_width => 2::integer, + n_distinct => 0.3::real); + +-- error: avg_width null +SELECT pg_catalog.pg_set_attribute_stats( + relation => 'stats_import.test'::regclass, + attname => 'id'::name, + inherited => false::boolean, + null_frac => 0.1::real, + avg_width => NULL::integer, + n_distinct => 0.3::real); + +-- error: avg_width null +SELECT pg_catalog.pg_set_attribute_stats( + relation => 'stats_import.test'::regclass, + attname => 'id'::name, + inherited => false::boolean, + null_frac => 0.1::real, + avg_width => 2::integer, + n_distinct => NULL::real); + +-- ok: no stakinds +SELECT pg_catalog.pg_set_attribute_stats( + relation => 'stats_import.test'::regclass, + attname => 'id'::name, + inherited => false::boolean, + null_frac => 0.1::real, + avg_width => 2::integer, + n_distinct => 0.3::real); + +SELECT stanullfrac, stawidth, stadistinct +FROM pg_statistic +WHERE starelid = 'stats_import.test'::regclass; + +-- error: mcv / mcf null mismatch +SELECT pg_catalog.pg_set_attribute_stats( + relation => 'stats_import.test'::regclass, + attname => 'id'::name, + inherited => false::boolean, + null_frac => 0.5::real, + avg_width => 2::integer, + n_distinct => -0.1::real, + most_common_freqs => '{0.1,0.2,0.3}'::real[] + ); + +-- error: mcv / mcf null mismatch part 2 +SELECT pg_catalog.pg_set_attribute_stats( + relation => 'stats_import.test'::regclass, + attname => 'id'::name, + inherited => false::boolean, + null_frac => 0.5::real, + avg_width => 2::integer, + n_distinct => -0.1::real, + most_common_vals => '{1,2,3}'::text + ); + +-- error: mcv / mcf type mismatch +SELECT pg_catalog.pg_set_attribute_stats( + relation => 'stats_import.test'::regclass, + attname => 'id'::name, + inherited => false::boolean, + null_frac => 0.5::real, + avg_width => 2::integer, + n_distinct => -0.1::real, + most_common_vals => '{2023-09-30,2024-10-31,3}'::text, + most_common_freqs => '{0.2,0.1}'::real[] + ); + +-- warning: mcv cast failure +SELECT pg_catalog.pg_set_attribute_stats( + relation => 'stats_import.test'::regclass, + attname => 'id'::name, + inherited => false::boolean, + null_frac => 0.5::real, + avg_width => 2::integer, + n_distinct => -0.1::real, + most_common_vals => '{2,four,3}'::text, + most_common_freqs => '{0.3,0.25,0.05}'::real[] + ); + +-- ok: mcv+mcf +SELECT pg_catalog.pg_set_attribute_stats( + relation => 'stats_import.test'::regclass, + attname => 'id'::name, + inherited => false::boolean, + null_frac => 0.5::real, + avg_width => 2::integer, + n_distinct => -0.1::real, + most_common_vals => '{2,1,3}'::text, + most_common_freqs => '{0.3,0.25,0.05}'::real[] + ); + +SELECT * +FROM pg_stats +WHERE schemaname = 'stats_import' +AND tablename = 'test' +AND inherited = false +AND attname = 'id'; + +-- error: histogram elements null value +-- this generates no warnings, but perhaps it should +SELECT pg_catalog.pg_set_attribute_stats( + relation => 'stats_import.test'::regclass, + attname => 'id'::name, + inherited => false::boolean, + null_frac => 0.5::real, + avg_width => 2::integer, + n_distinct => -0.1::real, + histogram_bounds => '{1,NULL,3,4}'::text + ); + +-- ok: histogram_bounds +SELECT pg_catalog.pg_set_attribute_stats( + relation => 'stats_import.test'::regclass, + attname => 'id'::name, + inherited => false::boolean, + null_frac => 0.5::real, + avg_width => 2::integer, + n_distinct => -0.1::real, + histogram_bounds => '{1,2,3,4}'::text + ); + +SELECT * +FROM pg_stats +WHERE schemaname = 'stats_import' +AND tablename = 'test' +AND inherited = false +AND attname = 'id'; + +-- ok: correlation +SELECT pg_catalog.pg_set_attribute_stats( + relation => 'stats_import.test'::regclass, + attname => 'id'::name, + inherited => false::boolean, + null_frac => 0.5::real, + avg_width => 2::integer, + n_distinct => -0.1::real, + correlation => 0.5::real); + +SELECT * +FROM pg_stats +WHERE schemaname = 'stats_import' +AND tablename = 'test' +AND inherited = false +AND attname = 'id'; + +-- error: scalars can't have mcelem +SELECT pg_catalog.pg_set_attribute_stats( + relation => 'stats_import.test'::regclass, + attname => 'id'::name, + inherited => false::boolean, + null_frac => 0.5::real, + avg_width => 2::integer, + n_distinct => -0.1::real, + most_common_elems => '{1,3}'::text, + most_common_elem_freqs => '{0.3,0.2,0.2,0.3,0.0}'::real[] + ); + +-- error: mcelem / mcelem mismatch +SELECT pg_catalog.pg_set_attribute_stats( + relation => 'stats_import.test'::regclass, + attname => 'tags'::name, + inherited => false::boolean, + null_frac => 0.5::real, + avg_width => 2::integer, + n_distinct => -0.1::real, + most_common_elems => '{one,two}'::text + ); + +-- error: mcelem / mcelem null mismatch part 2 +SELECT pg_catalog.pg_set_attribute_stats( + relation => 'stats_import.test'::regclass, + attname => 'tags'::name, + inherited => false::boolean, + null_frac => 0.5::real, + avg_width => 2::integer, + n_distinct => -0.1::real, + most_common_elem_freqs => '{0.3,0.2,0.2,0.3}'::real[] + ); + +-- ok: mcelem +SELECT pg_catalog.pg_set_attribute_stats( + relation => 'stats_import.test'::regclass, + attname => 'tags'::name, + inherited => false::boolean, + null_frac => 0.5::real, + avg_width => 2::integer, + n_distinct => -0.1::real, + most_common_elems => '{one,three}'::text, + most_common_elem_freqs => '{0.3,0.2,0.2,0.3,0.0}'::real[] + ); + +SELECT * +FROM pg_stats +WHERE schemaname = 'stats_import' +AND tablename = 'test' +AND inherited = false +AND attname = 'tags'; + +-- error: scalars can't have elem_count_histogram +SELECT pg_catalog.pg_set_attribute_stats( + relation => 'stats_import.test'::regclass, + attname => 'id'::name, + inherited => false::boolean, + null_frac => 0.5::real, + avg_width => 2::integer, + n_distinct => -0.1::real, + elem_count_histogram => '{1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}'::real[] + ); +-- error: elem_count_histogram null element +SELECT pg_catalog.pg_set_attribute_stats( + relation => 'stats_import.test'::regclass, + attname => 'tags'::name, + inherited => false::boolean, + null_frac => 0.5::real, + avg_width => 2::integer, + n_distinct => -0.1::real, + elem_count_histogram => '{1,1,NULL,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}'::real[] + ); +-- ok: elem_count_histogram +SELECT pg_catalog.pg_set_attribute_stats( + relation => 'stats_import.test'::regclass, + attname => 'tags'::name, + inherited => false::boolean, + null_frac => 0.5::real, + avg_width => 2::integer, + n_distinct => -0.1::real, + elem_count_histogram => '{1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}'::real[] + ); + +SELECT * +FROM pg_stats +WHERE schemaname = 'stats_import' +AND tablename = 'test' +AND inherited = false +AND attname = 'tags'; + +-- error: scalars can't have range stats +SELECT pg_catalog.pg_set_attribute_stats( + relation => 'stats_import.test'::regclass, + attname => 'id'::name, + inherited => false::boolean, + null_frac => 0.5::real, + avg_width => 2::integer, + n_distinct => -0.1::real, + range_empty_frac => 0.5::real, + range_length_histogram => '{399,499,Infinity}'::text + ); +-- error: range_empty_frac range_length_hist null mismatch +SELECT pg_catalog.pg_set_attribute_stats( + relation => 'stats_import.test'::regclass, + attname => 'arange'::name, + inherited => false::boolean, + null_frac => 0.5::real, + avg_width => 2::integer, + n_distinct => -0.1::real, + range_length_histogram => '{399,499,Infinity}'::text + ); +-- error: range_empty_frac range_length_hist null mismatch part 2 +SELECT pg_catalog.pg_set_attribute_stats( + relation => 'stats_import.test'::regclass, + attname => 'arange'::name, + inherited => false::boolean, + null_frac => 0.5::real, + avg_width => 2::integer, + n_distinct => -0.1::real, + range_empty_frac => 0.5::real + ); +-- ok: range_empty_frac + range_length_hist +SELECT pg_catalog.pg_set_attribute_stats( + relation => 'stats_import.test'::regclass, + attname => 'arange'::name, + inherited => false::boolean, + null_frac => 0.5::real, + avg_width => 2::integer, + n_distinct => -0.1::real, + range_empty_frac => 0.5::real, + range_length_histogram => '{399,499,Infinity}'::text + ); + +SELECT * +FROM pg_stats +WHERE schemaname = 'stats_import' +AND tablename = 'test' +AND inherited = false +AND attname = 'arange'; + +-- error: scalars can't have range stats +SELECT pg_catalog.pg_set_attribute_stats( + relation => 'stats_import.test'::regclass, + attname => 'id'::name, + inherited => false::boolean, + null_frac => 0.5::real, + avg_width => 2::integer, + n_distinct => -0.1::real, + range_bounds_histogram => '{"[-1,1)","[0,4)","[1,4)","[1,100)"}'::text + ); +-- ok: range_bounds_histogram +SELECT pg_catalog.pg_set_attribute_stats( + relation => 'stats_import.test'::regclass, + attname => 'arange'::name, + inherited => false::boolean, + null_frac => 0.5::real, + avg_width => 2::integer, + n_distinct => -0.1::real, + range_bounds_histogram => '{"[-1,1)","[0,4)","[1,4)","[1,100)"}'::text + ); + +SELECT * +FROM pg_stats +WHERE schemaname = 'stats_import' +AND tablename = 'test' +AND inherited = false +AND attname = 'arange'; + +-- error: cannot set most_common_elems for range type +SELECT pg_catalog.pg_set_attribute_stats( + relation => 'stats_import.test'::regclass, + attname => 'arange'::name, + inherited => false::boolean, + null_frac => 0.5::real, + avg_width => 2::integer, + n_distinct => -0.1::real, + most_common_vals => '{"[2,3)","[1,2)","[3,4)"}'::text, + most_common_freqs => '{0.3,0.25,0.05}'::real[], + histogram_bounds => '{"[1,2)","[2,3)","[3,4)","[4,5)"}'::text, + correlation => 1.1::real, + most_common_elems => '{3,1}'::text, + most_common_elem_freqs => '{0.3,0.2,0.2,0.3,0.0}'::real[], + range_empty_frac => -0.5::real, + range_length_histogram => '{399,499,Infinity}'::text, + range_bounds_histogram => '{"[-1,1)","[0,4)","[1,4)","[1,100)"}'::text + ); +-- +-- Test the ability to exactly copy data from one table to an identical table, +-- correctly reconstructing the stakind order as well as the staopN and +-- stacollN values. Because oids are not stable across databases, we can only +-- test this when the source and destination are on the same database +-- instance. For that reason, we borrow and adapt a query found in fe_utils +-- and used by pg_dump/pg_upgrade. +-- +INSERT INTO stats_import.test +SELECT 1, 'one', (1, 1.1, 'ONE', '2001-01-01', '{ "xkey": "xval" }')::stats_import.complex_type, int4range(1,4), array['red','green'] +UNION ALL +SELECT 2, 'two', (2, 2.2, 'TWO', '2002-02-02', '[true, 4, "six"]')::stats_import.complex_type, int4range(1,4), array['blue','yellow'] +UNION ALL +SELECT 3, 'tre', (3, 3.3, 'TRE', '2003-03-03', NULL)::stats_import.complex_type, int4range(-1,1), array['"orange"', 'purple', 'cyan'] +UNION ALL +SELECT 4, 'four', NULL, int4range(0,100), NULL; + +CREATE INDEX is_odd ON stats_import.test(((comp).a % 2 = 1)); + +-- Generate statistics on table with data +ANALYZE stats_import.test; + +CREATE TABLE stats_import.test_clone ( LIKE stats_import.test ) + WITH (autovacuum_enabled = false); + +CREATE INDEX is_odd_clone ON stats_import.test_clone(((comp).a % 2 = 1)); + +-- +-- Copy stats from test to test_clone, and is_odd to is_odd_clone +-- +SELECT s.schemaname, s.tablename, s.attname, s.inherited +FROM pg_catalog.pg_stats AS s +CROSS JOIN LATERAL + pg_catalog.pg_set_attribute_stats( + relation => ('stats_import.' || s.tablename || '_clone')::regclass::oid, + attname => s.attname, + inherited => s.inherited, + null_frac => s.null_frac, + avg_width => s.avg_width, + n_distinct => s.n_distinct, + most_common_vals => s.most_common_vals::text, + most_common_freqs => s.most_common_freqs, + histogram_bounds => s.histogram_bounds::text, + correlation => s.correlation, + most_common_elems => s.most_common_elems::text, + most_common_elem_freqs => s.most_common_elem_freqs, + elem_count_histogram => s.elem_count_histogram, + range_bounds_histogram => s.range_bounds_histogram::text, + range_empty_frac => s.range_empty_frac, + range_length_histogram => s.range_length_histogram::text) AS r +WHERE s.schemaname = 'stats_import' +AND s.tablename IN ('test', 'is_odd') +ORDER BY s.tablename, s.attname, s.inherited; + +SELECT c.relname, COUNT(*) AS num_stats +FROM pg_class AS c +JOIN pg_statistic s ON s.starelid = c.oid +WHERE c.relnamespace = 'stats_import'::regnamespace +AND c.relname IN ('test', 'test_clone', 'is_odd', 'is_odd_clone') +GROUP BY c.relname +ORDER BY c.relname; + +-- check test minus test_clone +SELECT + a.attname, s.stainherit, s.stanullfrac, s.stawidth, s.stadistinct, + s.stakind1, s.stakind2, s.stakind3, s.stakind4, s.stakind5, + s.staop1, s.staop2, s.staop3, s.staop4, s.staop5, + s.stacoll1, s.stacoll2, s.stacoll3, s.stacoll4, s.stacoll5, + s.stanumbers1, s.stanumbers2, s.stanumbers3, s.stanumbers4, s.stanumbers5, + s.stavalues1::text AS sv1, s.stavalues2::text AS sv2, + s.stavalues3::text AS sv3, s.stavalues4::text AS sv4, + s.stavalues5::text AS sv5, 'test' AS direction +FROM pg_statistic s +JOIN pg_attribute a ON a.attrelid = s.starelid AND a.attnum = s.staattnum +WHERE s.starelid = 'stats_import.test'::regclass +EXCEPT +SELECT + a.attname, s.stainherit, s.stanullfrac, s.stawidth, s.stadistinct, + s.stakind1, s.stakind2, s.stakind3, s.stakind4, s.stakind5, + s.staop1, s.staop2, s.staop3, s.staop4, s.staop5, + s.stacoll1, s.stacoll2, s.stacoll3, s.stacoll4, s.stacoll5, + s.stanumbers1, s.stanumbers2, s.stanumbers3, s.stanumbers4, s.stanumbers5, + s.stavalues1::text AS sv1, s.stavalues2::text AS sv2, + s.stavalues3::text AS sv3, s.stavalues4::text AS sv4, + s.stavalues5::text AS sv5, 'test' AS direction +FROM pg_statistic s +JOIN pg_attribute a ON a.attrelid = s.starelid AND a.attnum = s.staattnum +WHERE s.starelid = 'stats_import.test_clone'::regclass; + +-- check test_clone minus test +SELECT + a.attname, s.stainherit, s.stanullfrac, s.stawidth, s.stadistinct, + s.stakind1, s.stakind2, s.stakind3, s.stakind4, s.stakind5, + s.staop1, s.staop2, s.staop3, s.staop4, s.staop5, + s.stacoll1, s.stacoll2, s.stacoll3, s.stacoll4, s.stacoll5, + s.stanumbers1, s.stanumbers2, s.stanumbers3, s.stanumbers4, s.stanumbers5, + s.stavalues1::text AS sv1, s.stavalues2::text AS sv2, + s.stavalues3::text AS sv3, s.stavalues4::text AS sv4, + s.stavalues5::text AS sv5, 'test_clone' AS direction +FROM pg_statistic s +JOIN pg_attribute a ON a.attrelid = s.starelid AND a.attnum = s.staattnum +WHERE s.starelid = 'stats_import.test_clone'::regclass +EXCEPT +SELECT + a.attname, s.stainherit, s.stanullfrac, s.stawidth, s.stadistinct, + s.stakind1, s.stakind2, s.stakind3, s.stakind4, s.stakind5, + s.staop1, s.staop2, s.staop3, s.staop4, s.staop5, + s.stacoll1, s.stacoll2, s.stacoll3, s.stacoll4, s.stacoll5, + s.stanumbers1, s.stanumbers2, s.stanumbers3, s.stanumbers4, s.stanumbers5, + s.stavalues1::text AS sv1, s.stavalues2::text AS sv2, + s.stavalues3::text AS sv3, s.stavalues4::text AS sv4, + s.stavalues5::text AS sv5, 'test_clone' AS direction +FROM pg_statistic s +JOIN pg_attribute a ON a.attrelid = s.starelid AND a.attnum = s.staattnum +WHERE s.starelid = 'stats_import.test'::regclass; + +-- check is_odd minus is_odd_clone +SELECT + a.attname, s.stainherit, s.stanullfrac, s.stawidth, s.stadistinct, + s.stakind1, s.stakind2, s.stakind3, s.stakind4, s.stakind5, + s.staop1, s.staop2, s.staop3, s.staop4, s.staop5, + s.stacoll1, s.stacoll2, s.stacoll3, s.stacoll4, s.stacoll5, + s.stanumbers1, s.stanumbers2, s.stanumbers3, s.stanumbers4, s.stanumbers5, + s.stavalues1::text AS sv1, s.stavalues2::text AS sv2, + s.stavalues3::text AS sv3, s.stavalues4::text AS sv4, + s.stavalues5::text AS sv5, 'is_odd' AS direction +FROM pg_statistic s +JOIN pg_attribute a ON a.attrelid = s.starelid AND a.attnum = s.staattnum +WHERE s.starelid = 'stats_import.is_odd'::regclass +EXCEPT +SELECT + a.attname, s.stainherit, s.stanullfrac, s.stawidth, s.stadistinct, + s.stakind1, s.stakind2, s.stakind3, s.stakind4, s.stakind5, + s.staop1, s.staop2, s.staop3, s.staop4, s.staop5, + s.stacoll1, s.stacoll2, s.stacoll3, s.stacoll4, s.stacoll5, + s.stanumbers1, s.stanumbers2, s.stanumbers3, s.stanumbers4, s.stanumbers5, + s.stavalues1::text AS sv1, s.stavalues2::text AS sv2, + s.stavalues3::text AS sv3, s.stavalues4::text AS sv4, + s.stavalues5::text AS sv5, 'is_odd' AS direction +FROM pg_statistic s +JOIN pg_attribute a ON a.attrelid = s.starelid AND a.attnum = s.staattnum +WHERE s.starelid = 'stats_import.is_odd_clone'::regclass; + +-- check is_odd_clone minus is_odd +SELECT + a.attname, s.stainherit, s.stanullfrac, s.stawidth, s.stadistinct, + s.stakind1, s.stakind2, s.stakind3, s.stakind4, s.stakind5, + s.staop1, s.staop2, s.staop3, s.staop4, s.staop5, + s.stacoll1, s.stacoll2, s.stacoll3, s.stacoll4, s.stacoll5, + s.stanumbers1, s.stanumbers2, s.stanumbers3, s.stanumbers4, s.stanumbers5, + s.stavalues1::text AS sv1, s.stavalues2::text AS sv2, + s.stavalues3::text AS sv3, s.stavalues4::text AS sv4, + s.stavalues5::text AS sv5, 'is_odd_clone' AS direction +FROM pg_statistic s +JOIN pg_attribute a ON a.attrelid = s.starelid AND a.attnum = s.staattnum +WHERE s.starelid = 'stats_import.is_odd_clone'::regclass +EXCEPT +SELECT + a.attname, s.stainherit, s.stanullfrac, s.stawidth, s.stadistinct, + s.stakind1, s.stakind2, s.stakind3, s.stakind4, s.stakind5, + s.staop1, s.staop2, s.staop3, s.staop4, s.staop5, + s.stacoll1, s.stacoll2, s.stacoll3, s.stacoll4, s.stacoll5, + s.stanumbers1, s.stanumbers2, s.stanumbers3, s.stanumbers4, s.stanumbers5, + s.stavalues1::text AS sv1, s.stavalues2::text AS sv2, + s.stavalues3::text AS sv3, s.stavalues4::text AS sv4, + s.stavalues5::text AS sv5, 'is_odd_clone' AS direction +FROM pg_statistic s +JOIN pg_attribute a ON a.attrelid = s.starelid AND a.attnum = s.staattnum +WHERE s.starelid = 'stats_import.is_odd'::regclass; + DROP SCHEMA stats_import CASCADE; From d74f298be2e61bd395d1f3a74403bd7bcab6ebc5 Mon Sep 17 00:00:00 2001 From: Jeff Davis Date: Wed, 23 Oct 2024 16:11:45 -0700 Subject: [PATCH 08/54] Improve pg_set_attribute_stats() error message. Previously, an invalid attribute name was caught, but the error message was unhelpful. --- src/backend/statistics/attribute_stats.c | 10 ++++++++++ src/test/regress/expected/stats_import.out | 23 +++++++++++++++++++++- src/test/regress/sql/stats_import.sql | 23 +++++++++++++++++++++- 3 files changed, 54 insertions(+), 2 deletions(-) diff --git a/src/backend/statistics/attribute_stats.c b/src/backend/statistics/attribute_stats.c index c920409680a..e7c3bda99ea 100644 --- a/src/backend/statistics/attribute_stats.c +++ b/src/backend/statistics/attribute_stats.c @@ -161,6 +161,11 @@ attribute_statistics_update(FunctionCallInfo fcinfo, int elevel) stats_check_required_arg(fcinfo, attarginfo, ATTNAME_ARG); attname = PG_GETARG_NAME(ATTNAME_ARG); attnum = get_attnum(reloid, NameStr(*attname)); + if (attnum == InvalidAttrNumber) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_COLUMN), + errmsg("column \"%s\" of relation \"%s\" does not exist", + NameStr(*attname), get_rel_name(reloid)))); stats_check_required_arg(fcinfo, attarginfo, INHERITED_ARG); inherited = PG_GETARG_BOOL(INHERITED_ARG); @@ -860,6 +865,11 @@ pg_clear_attribute_stats(PG_FUNCTION_ARGS) stats_check_required_arg(fcinfo, attarginfo, ATTNAME_ARG); attname = PG_GETARG_NAME(ATTNAME_ARG); attnum = get_attnum(reloid, NameStr(*attname)); + if (attnum == InvalidAttrNumber) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_COLUMN), + errmsg("column \"%s\" of relation \"%s\" does not exist", + NameStr(*attname), get_rel_name(reloid)))); stats_check_required_arg(fcinfo, attarginfo, INHERITED_ARG); inherited = PG_GETARG_BOOL(INHERITED_ARG); diff --git a/src/test/regress/expected/stats_import.out b/src/test/regress/expected/stats_import.out index 2868cc47f1d..6ab21ca324c 100644 --- a/src/test/regress/expected/stats_import.out +++ b/src/test/regress/expected/stats_import.out @@ -180,6 +180,12 @@ SELECT pg_catalog.pg_set_attribute_stats( avg_width => 2::integer, n_distinct => 0.3::real); ERROR: could not open relation with OID 0 +-- error: object doesn't exist +SELECT pg_catalog.pg_clear_attribute_stats( + relation => '0'::oid, + attname => 'id'::name, + inherited => false::boolean); +ERROR: could not open relation with OID 0 -- error: relation null SELECT pg_catalog.pg_set_attribute_stats( relation => NULL::oid, @@ -189,6 +195,21 @@ SELECT pg_catalog.pg_set_attribute_stats( avg_width => 2::integer, n_distinct => 0.3::real); ERROR: "relation" cannot be NULL +-- error: attname doesn't exist +SELECT pg_catalog.pg_set_attribute_stats( + relation => 'stats_import.test'::regclass, + attname => 'nope'::name, + inherited => false::boolean, + null_frac => 0.1::real, + avg_width => 2::integer, + n_distinct => 0.3::real); +ERROR: column "nope" of relation "test" does not exist +-- error: attname doesn't exist +SELECT pg_catalog.pg_clear_attribute_stats( + relation => 'stats_import.test'::regclass, + attname => 'nope'::name, + inherited => false::boolean); +ERROR: column "nope" of relation "test" does not exist -- error: attname null SELECT pg_catalog.pg_set_attribute_stats( relation => 'stats_import.test'::regclass, @@ -301,7 +322,7 @@ SELECT pg_catalog.pg_set_attribute_stats( most_common_freqs => '{0.2,0.1}'::real[] ); ERROR: invalid input syntax for type integer: "2023-09-30" --- warning: mcv cast failure +-- error: mcv cast failure SELECT pg_catalog.pg_set_attribute_stats( relation => 'stats_import.test'::regclass, attname => 'id'::name, diff --git a/src/test/regress/sql/stats_import.sql b/src/test/regress/sql/stats_import.sql index 9b6c9094862..b5689ac8fb3 100644 --- a/src/test/regress/sql/stats_import.sql +++ b/src/test/regress/sql/stats_import.sql @@ -130,6 +130,12 @@ SELECT pg_catalog.pg_set_attribute_stats( avg_width => 2::integer, n_distinct => 0.3::real); +-- error: object doesn't exist +SELECT pg_catalog.pg_clear_attribute_stats( + relation => '0'::oid, + attname => 'id'::name, + inherited => false::boolean); + -- error: relation null SELECT pg_catalog.pg_set_attribute_stats( relation => NULL::oid, @@ -139,6 +145,21 @@ SELECT pg_catalog.pg_set_attribute_stats( avg_width => 2::integer, n_distinct => 0.3::real); +-- error: attname doesn't exist +SELECT pg_catalog.pg_set_attribute_stats( + relation => 'stats_import.test'::regclass, + attname => 'nope'::name, + inherited => false::boolean, + null_frac => 0.1::real, + avg_width => 2::integer, + n_distinct => 0.3::real); + +-- error: attname doesn't exist +SELECT pg_catalog.pg_clear_attribute_stats( + relation => 'stats_import.test'::regclass, + attname => 'nope'::name, + inherited => false::boolean); + -- error: attname null SELECT pg_catalog.pg_set_attribute_stats( relation => 'stats_import.test'::regclass, @@ -231,7 +252,7 @@ SELECT pg_catalog.pg_set_attribute_stats( most_common_freqs => '{0.2,0.1}'::real[] ); --- warning: mcv cast failure +-- error: mcv cast failure SELECT pg_catalog.pg_set_attribute_stats( relation => 'stats_import.test'::regclass, attname => 'id'::name, From 85d2600b621fb4e3a5506daa326afe3eb11a2913 Mon Sep 17 00:00:00 2001 From: Jeff Davis Date: Thu, 24 Oct 2024 12:08:00 -0700 Subject: [PATCH 09/54] Add functions pg_restore_relation_stats(), pg_restore_attribute_stats(). Similar to the pg_set_*_stats() functions, except with a variadic signature that's designed to be more future-proof. Additionally, most problems are reported as WARNINGs rather than ERRORs, allowing most stats to be restored even if some cannot. These functions are intended to be called from pg_dump to avoid the need to run ANALYZE after an upgrade. Author: Corey Huinker Discussion: https://postgr.es/m/CADkLM=eErgzn7ECDpwFcptJKOk9SxZEk5Pot4d94eVTZsvj3gw@mail.gmail.com --- doc/src/sgml/func.sgml | 100 +++ src/backend/statistics/attribute_stats.c | 19 + src/backend/statistics/relation_stats.c | 60 +- src/backend/statistics/stat_utils.c | 126 +++ src/include/catalog/pg_proc.dat | 20 +- src/include/statistics/stat_utils.h | 5 + src/test/regress/expected/stats_import.out | 955 ++++++++++++++++++++- src/test/regress/sql/stats_import.sql | 696 ++++++++++++++- 8 files changed, 1890 insertions(+), 91 deletions(-) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 817669ed896..8f22eb40cf3 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -30038,6 +30038,55 @@ postgres=# SELECT '0/0'::pg_lsn + pd.segment_number * ps.setting::int + :offset + + + + pg_restore_relation_stats + + pg_restore_relation_stats ( + VARIADIC kwargs "any" ) + boolean + + + Similar to pg_set_relation_stats(), but intended + for bulk restore of relation statistics. The tracked statistics may + change from version to version, so the primary purpose of this + function is to maintain a consistent function signature to avoid + errors when restoring statistics from previous versions. + + + Arguments are passed as pairs of argname + and argvalue, where + argname corresponds to a named argument in + pg_set_relation_stats() and + argvalue is of the corresponding type. + + + Additionally, this function supports argument name + version of type integer, which + specifies the version from which the statistics originated, improving + intepretation of older statistics. + + + For example, to set the relpages and + reltuples of the table + mytable: + + SELECT pg_restore_relation_stats( + 'relation', 'mytable'::regclass, + 'relpages', 173::integer, + 'reltuples', 10000::float4); + + + + Minor errors are reported as a WARNING and + ignored, and remaining statistics will still be restored. If all + specified statistics are successfully restored, return + true, otherwise false. + + + + @@ -30109,6 +30158,57 @@ postgres=# SELECT '0/0'::pg_lsn + pd.segment_number * ps.setting::int + :offset + + + + pg_restore_attribute_stats + + pg_restore_attribute_stats ( + VARIADIC kwargs "any" ) + boolean + + + Similar to pg_set_attribute_stats(), but + intended for bulk restore of attribute statistics. The tracked + statistics may change from version to version, so the primary purpose + of this function is to maintain a consistent function signature to + avoid errors when restoring statistics from previous versions. + + + Arguments are passed as pairs of argname + and argvalue, where + argname corresponds to a named argument in + pg_set_attribute_stats() and + argvalue is of the corresponding type. + + + Additionally, this function supports argument name + version of type integer, which + specifies the version from which the statistics originated, improving + intepretation of older statistics. + + + For example, to set the avg_width and + null_frac for the attribute + col1 of the table + mytable: + + SELECT pg_restore_attribute_stats( + 'relation', 'mytable'::regclass, + 'attname', 'col1'::name, + 'inherited', false, + 'avg_width', 125::integer, + 'null_frac', 0.5::real); + + + + Minor errors are reported as a WARNING and + ignored, and remaining statistics will still be restored. If all + specified statistics are successfully restored, return + true, otherwise false. + + + diff --git a/src/backend/statistics/attribute_stats.c b/src/backend/statistics/attribute_stats.c index e7c3bda99ea..0da3bc2e3a7 100644 --- a/src/backend/statistics/attribute_stats.c +++ b/src/backend/statistics/attribute_stats.c @@ -877,3 +877,22 @@ pg_clear_attribute_stats(PG_FUNCTION_ARGS) delete_pg_statistic(reloid, attnum, inherited); PG_RETURN_VOID(); } + +Datum +pg_restore_attribute_stats(PG_FUNCTION_ARGS) +{ + LOCAL_FCINFO(positional_fcinfo, NUM_ATTRIBUTE_STATS_ARGS); + bool result = true; + + InitFunctionCallInfoData(*positional_fcinfo, NULL, NUM_ATTRIBUTE_STATS_ARGS, + InvalidOid, NULL, NULL); + + if (!stats_fill_fcinfo_from_arg_pairs(fcinfo, positional_fcinfo, + attarginfo, WARNING)) + result = false; + + if (!attribute_statistics_update(positional_fcinfo, WARNING)) + result = false; + + PG_RETURN_BOOL(result); +} diff --git a/src/backend/statistics/relation_stats.c b/src/backend/statistics/relation_stats.c index b1eb8a9bbaf..5a2aabc921e 100644 --- a/src/backend/statistics/relation_stats.c +++ b/src/backend/statistics/relation_stats.c @@ -67,8 +67,7 @@ relation_statistics_update(FunctionCallInfo fcinfo, int elevel) bool nulls[3] = {0}; int ncols = 0; TupleDesc tupdesc; - HeapTuple newtup; - + bool result = true; stats_check_required_arg(fcinfo, relarginfo, RELATION_ARG); reloid = PG_GETARG_OID(RELATION_ARG); @@ -109,11 +108,9 @@ relation_statistics_update(FunctionCallInfo fcinfo, int elevel) ereport(elevel, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("relpages cannot be < -1"))); - table_close(crel, RowExclusiveLock); - return false; + result = false; } - - if (relpages != pgcform->relpages) + else if (relpages != pgcform->relpages) { replaces[ncols] = Anum_pg_class_relpages; values[ncols] = Int32GetDatum(relpages); @@ -130,16 +127,15 @@ relation_statistics_update(FunctionCallInfo fcinfo, int elevel) ereport(elevel, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("reltuples cannot be < -1.0"))); - table_close(crel, RowExclusiveLock); - return false; + result = false; } - - if (reltuples != pgcform->reltuples) + else if (reltuples != pgcform->reltuples) { replaces[ncols] = Anum_pg_class_reltuples; values[ncols] = Float4GetDatum(reltuples); ncols++; } + } if (!PG_ARGISNULL(RELALLVISIBLE_ARG)) @@ -151,11 +147,9 @@ relation_statistics_update(FunctionCallInfo fcinfo, int elevel) ereport(elevel, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("relallvisible cannot be < 0"))); - table_close(crel, RowExclusiveLock); - return false; + result = false; } - - if (relallvisible != pgcform->relallvisible) + else if (relallvisible != pgcform->relallvisible) { replaces[ncols] = Anum_pg_class_relallvisible; values[ncols] = Int32GetDatum(relallvisible); @@ -164,22 +158,20 @@ relation_statistics_update(FunctionCallInfo fcinfo, int elevel) } /* only update pg_class if there is a meaningful change */ - if (ncols == 0) + if (ncols > 0) { - table_close(crel, RowExclusiveLock); - return false; - } - - newtup = heap_modify_tuple_by_cols(ctup, tupdesc, ncols, replaces, values, - nulls); + HeapTuple newtup; - CatalogTupleUpdate(crel, &newtup->t_self, newtup); - heap_freetuple(newtup); + newtup = heap_modify_tuple_by_cols(ctup, tupdesc, ncols, replaces, values, + nulls); + CatalogTupleUpdate(crel, &newtup->t_self, newtup); + heap_freetuple(newtup); + } /* release the lock, consistent with vac_update_relstats() */ table_close(crel, RowExclusiveLock); - return true; + return result; } /* @@ -215,3 +207,23 @@ pg_clear_relation_stats(PG_FUNCTION_ARGS) relation_statistics_update(newfcinfo, ERROR); PG_RETURN_VOID(); } + +Datum +pg_restore_relation_stats(PG_FUNCTION_ARGS) +{ + LOCAL_FCINFO(positional_fcinfo, NUM_RELATION_STATS_ARGS); + bool result = true; + + InitFunctionCallInfoData(*positional_fcinfo, NULL, + NUM_RELATION_STATS_ARGS, + InvalidOid, NULL, NULL); + + if (!stats_fill_fcinfo_from_arg_pairs(fcinfo, positional_fcinfo, + relarginfo, WARNING)) + result = false; + + if (!relation_statistics_update(positional_fcinfo, WARNING)) + result = false; + + PG_RETURN_BOOL(result); +} diff --git a/src/backend/statistics/stat_utils.c b/src/backend/statistics/stat_utils.c index e7997ec486e..740038e3fe7 100644 --- a/src/backend/statistics/stat_utils.c +++ b/src/backend/statistics/stat_utils.c @@ -18,6 +18,7 @@ #include "access/relation.h" #include "catalog/pg_database.h" +#include "funcapi.h" #include "miscadmin.h" #include "statistics/stat_utils.h" #include "utils/array.h" @@ -165,3 +166,128 @@ stats_lock_check_privileges(Oid reloid) relation_close(rel, NoLock); } + +/* + * Find the argument number for the given argument name, returning -1 if not + * found. + */ +static int +get_arg_by_name(const char *argname, struct StatsArgInfo *arginfo, int elevel) +{ + int argnum; + + for (argnum = 0; arginfo[argnum].argname != NULL; argnum++) + if (pg_strcasecmp(argname, arginfo[argnum].argname) == 0) + return argnum; + + ereport(elevel, + (errmsg("unrecognized argument name: \"%s\"", argname))); + + return -1; +} + +/* + * Ensure that a given argument matched the expected type. + */ +static bool +stats_check_arg_type(const char *argname, Oid argtype, Oid expectedtype, int elevel) +{ + if (argtype != expectedtype) + { + ereport(elevel, + (errmsg("argument \"%s\" has type \"%s\", expected type \"%s\"", + argname, format_type_be(argtype), + format_type_be(expectedtype)))); + return false; + } + + return true; +} + +/* + * Translate variadic argument pairs from 'pairs_fcinfo' into a + * 'positional_fcinfo' appropriate for calling relation_statistics_update() or + * attribute_statistics_update() with positional arguments. + * + * Caller should have already initialized positional_fcinfo with a size + * appropriate for calling the intended positional function, and arginfo + * should also match the intended positional function. + */ +bool +stats_fill_fcinfo_from_arg_pairs(FunctionCallInfo pairs_fcinfo, + FunctionCallInfo positional_fcinfo, + struct StatsArgInfo *arginfo, + int elevel) +{ + Datum *args; + bool *argnulls; + Oid *types; + int nargs; + bool result = true; + + /* clear positional args */ + for (int i = 0; arginfo[i].argname != NULL; i++) + { + positional_fcinfo->args[i].value = (Datum) 0; + positional_fcinfo->args[i].isnull = true; + } + + nargs = extract_variadic_args(pairs_fcinfo, 0, true, + &args, &types, &argnulls); + + if (nargs % 2 != 0) + ereport(ERROR, + errmsg("variadic arguments must be name/value pairs"), + errhint("Provide an even number of variadic arguments that can be divided into pairs.")); + + /* + * For each argument name/value pair, find corresponding positional + * argument for the argument name, and assign the argument value to + * postitional_fcinfo. + */ + for (int i = 0; i < nargs; i += 2) + { + int argnum; + char *argname; + + if (argnulls[i]) + ereport(ERROR, + (errmsg("name at variadic position %d is NULL", i + 1))); + + if (types[i] != TEXTOID) + ereport(ERROR, + (errmsg("name at variadic position %d has type \"%s\", expected type \"%s\"", + i + 1, format_type_be(types[i]), + format_type_be(TEXTOID)))); + + if (argnulls[i + 1]) + continue; + + argname = TextDatumGetCString(args[i]); + + /* + * The 'version' argument is a special case, not handled by arginfo + * because it's not a valid positional argument. + * + * For now, 'version' is accepted but ignored. In the future it can be + * used to interpret older statistics properly. + */ + if (pg_strcasecmp(argname, "version") == 0) + continue; + + argnum = get_arg_by_name(argname, arginfo, elevel); + + if (argnum < 0 || !stats_check_arg_type(argname, types[i + 1], + arginfo[argnum].argtype, + elevel)) + { + result = false; + continue; + } + + positional_fcinfo->args[argnum].value = args[i + 1]; + positional_fcinfo->args[argnum].isnull = false; + } + + return result; +} diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 429c195e652..bac69bcf687 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -12184,7 +12184,25 @@ proallargtypes => '{int8,pg_lsn,pg_lsn,int4}', proargmodes => '{o,o,o,o}', proargnames => '{summarized_tli,summarized_lsn,pending_lsn,summarizer_pid}', prosrc => 'pg_get_wal_summarizer_state' }, - # Statistics Import +# Statistics Import +{ oid => '6363', + descr => 'restore statistics on relation', + proname => 'pg_restore_relation_stats', provolatile => 'v', proisstrict => 'f', + provariadic => 'any', + proparallel => 'u', prorettype => 'bool', + proargtypes => 'any', + proargnames => '{kwargs}', + proargmodes => '{v}', + prosrc => 'pg_restore_relation_stats' }, +{ oid => '6398', + descr => 'restore statistics on attribute', + proname => 'pg_restore_attribute_stats', provolatile => 'v', proisstrict => 'f', + provariadic => 'any', + proparallel => 'u', prorettype => 'bool', + proargtypes => 'any', + proargnames => '{kwargs}', + proargmodes => '{v}', + prosrc => 'pg_restore_attribute_stats' }, { oid => '9162', descr => 'set statistics on attribute', proname => 'pg_set_attribute_stats', provolatile => 'v', proisstrict => 'f', diff --git a/src/include/statistics/stat_utils.h b/src/include/statistics/stat_utils.h index e5f19d00609..61f1f6b3108 100644 --- a/src/include/statistics/stat_utils.h +++ b/src/include/statistics/stat_utils.h @@ -33,4 +33,9 @@ extern bool stats_check_arg_pair(FunctionCallInfo fcinfo, extern void stats_lock_check_privileges(Oid reloid); +extern bool stats_fill_fcinfo_from_arg_pairs(FunctionCallInfo pairs_fcinfo, + FunctionCallInfo positional_fcinfo, + struct StatsArgInfo *arginfo, + int elevel); + #endif /* STATS_UTILS_H */ diff --git a/src/test/regress/expected/stats_import.out b/src/test/regress/expected/stats_import.out index 6ab21ca324c..9186fc01ecc 100644 --- a/src/test/regress/expected/stats_import.out +++ b/src/test/regress/expected/stats_import.out @@ -228,45 +228,6 @@ SELECT pg_catalog.pg_set_attribute_stats( avg_width => 2::integer, n_distinct => 0.3::real); ERROR: "inherited" cannot be NULL --- error: null_frac null -SELECT pg_catalog.pg_set_attribute_stats( - relation => 'stats_import.test'::regclass, - attname => 'id'::name, - inherited => false::boolean, - null_frac => NULL::real, - avg_width => 2::integer, - n_distinct => 0.3::real); - pg_set_attribute_stats ------------------------- - -(1 row) - --- error: avg_width null -SELECT pg_catalog.pg_set_attribute_stats( - relation => 'stats_import.test'::regclass, - attname => 'id'::name, - inherited => false::boolean, - null_frac => 0.1::real, - avg_width => NULL::integer, - n_distinct => 0.3::real); - pg_set_attribute_stats ------------------------- - -(1 row) - --- error: avg_width null -SELECT pg_catalog.pg_set_attribute_stats( - relation => 'stats_import.test'::regclass, - attname => 'id'::name, - inherited => false::boolean, - null_frac => 0.1::real, - avg_width => 2::integer, - n_distinct => NULL::real); - pg_set_attribute_stats ------------------------- - -(1 row) - -- ok: no stakinds SELECT pg_catalog.pg_set_attribute_stats( relation => 'stats_import.test'::regclass, @@ -655,6 +616,711 @@ SELECT pg_catalog.pg_set_attribute_stats( ); ERROR: unable to determine element type of attribute "arange" DETAIL: Cannot set STATISTIC_KIND_MCELEM or STATISTIC_KIND_DECHIST. +-- +-- Clear attribute stats to try again with restore functions +-- (relation stats were already cleared). +-- +SELECT + pg_catalog.pg_clear_attribute_stats( + 'stats_import.test'::regclass, + s.attname, + s.inherited) +FROM pg_catalog.pg_stats AS s +WHERE s.schemaname = 'stats_import' +AND s.tablename = 'test' +ORDER BY s.attname, s.inherited; + pg_clear_attribute_stats +-------------------------- + + + +(3 rows) + +-- reject: argument name is NULL +SELECT pg_restore_relation_stats( + 'relation', '0'::oid::regclass, + 'version', 150000::integer, + NULL, '17'::integer, + 'reltuples', 400::real, + 'relallvisible', 4::integer); +ERROR: name at variadic position 5 is NULL +-- reject: argument name is an integer +SELECT pg_restore_relation_stats( + 'relation', '0'::oid::regclass, + 'version', 150000::integer, + 17, '17'::integer, + 'reltuples', 400::real, + 'relallvisible', 4::integer); +ERROR: name at variadic position 5 has type "integer", expected type "text" +-- reject: odd number of variadic arguments cannot be pairs +SELECT pg_restore_relation_stats( + 'relation', '0'::oid::regclass, + 'version', 150000::integer, + 'relpages', '17'::integer, + 'reltuples', 400::real, + 'relallvisible'); +ERROR: variadic arguments must be name/value pairs +HINT: Provide an even number of variadic arguments that can be divided into pairs. +-- reject: object doesn't exist +SELECT pg_restore_relation_stats( + 'relation', '0'::oid::regclass, + 'version', 150000::integer, + 'relpages', '17'::integer, + 'reltuples', 400::real, + 'relallvisible', 4::integer); +ERROR: could not open relation with OID 0 +-- ok: set all stats +SELECT pg_restore_relation_stats( + 'relation', 'stats_import.test'::regclass, + 'version', 150000::integer, + 'relpages', '17'::integer, + 'reltuples', 400::real, + 'relallvisible', 4::integer); + pg_restore_relation_stats +--------------------------- + t +(1 row) + +SELECT relpages, reltuples, relallvisible +FROM pg_class +WHERE oid = 'stats_import.test'::regclass; + relpages | reltuples | relallvisible +----------+-----------+--------------- + 17 | 400 | 4 +(1 row) + +-- ok: just relpages +SELECT pg_restore_relation_stats( + 'relation', 'stats_import.test'::regclass, + 'version', 150000::integer, + 'relpages', '16'::integer); + pg_restore_relation_stats +--------------------------- + t +(1 row) + +SELECT relpages, reltuples, relallvisible +FROM pg_class +WHERE oid = 'stats_import.test'::regclass; + relpages | reltuples | relallvisible +----------+-----------+--------------- + 16 | 400 | 4 +(1 row) + +-- ok: just reltuples +SELECT pg_restore_relation_stats( + 'relation', 'stats_import.test'::regclass, + 'version', 150000::integer, + 'reltuples', '500'::real); + pg_restore_relation_stats +--------------------------- + t +(1 row) + +SELECT relpages, reltuples, relallvisible +FROM pg_class +WHERE oid = 'stats_import.test'::regclass; + relpages | reltuples | relallvisible +----------+-----------+--------------- + 16 | 500 | 4 +(1 row) + +-- ok: just relallvisible +SELECT pg_restore_relation_stats( + 'relation', 'stats_import.test'::regclass, + 'version', 150000::integer, + 'relallvisible', 5::integer); + pg_restore_relation_stats +--------------------------- + t +(1 row) + +SELECT relpages, reltuples, relallvisible +FROM pg_class +WHERE oid = 'stats_import.test'::regclass; + relpages | reltuples | relallvisible +----------+-----------+--------------- + 16 | 500 | 5 +(1 row) + +-- warn and error: unrecognized argument name +SELECT pg_restore_relation_stats( + 'relation', '0'::oid::regclass, + 'version', 150000::integer, + 'relpages', '17'::integer, + 'reltuples', 400::real, + 'nope', 4::integer); +WARNING: unrecognized argument name: "nope" +ERROR: could not open relation with OID 0 +-- warn: bad relpages type +SELECT pg_restore_relation_stats( + 'relation', 'stats_import.test'::regclass, + 'version', 150000::integer, + 'relpages', 'nope'::text, + 'reltuples', 400.0::real, + 'relallvisible', 4::integer); +WARNING: argument "relpages" has type "text", expected type "integer" + pg_restore_relation_stats +--------------------------- + f +(1 row) + +SELECT relpages, reltuples, relallvisible +FROM pg_class +WHERE oid = 'stats_import.test'::regclass; + relpages | reltuples | relallvisible +----------+-----------+--------------- + 16 | 400 | 4 +(1 row) + +-- error: object does not exist +SELECT pg_catalog.pg_restore_attribute_stats( + 'relation', '0'::oid::regclass, + 'attname', 'id'::name, + 'inherited', false::boolean, + 'version', 150000::integer, + 'null_frac', 0.1::real, + 'avg_width', 2::integer, + 'n_distinct', 0.3::real); +ERROR: could not open relation with OID 0 +-- error: relation null +SELECT pg_catalog.pg_restore_attribute_stats( + 'relation', NULL::oid, + 'attname', 'id'::name, + 'inherited', false::boolean, + 'version', 150000::integer, + 'null_frac', 0.1::real, + 'avg_width', 2::integer, + 'n_distinct', 0.3::real); +ERROR: "relation" cannot be NULL +-- error: attname null +SELECT pg_catalog.pg_restore_attribute_stats( + 'relation', 'stats_import.test'::regclass, + 'attname', NULL::name, + 'inherited', false::boolean, + 'version', 150000::integer, + 'null_frac', 0.1::real, + 'avg_width', 2::integer, + 'n_distinct', 0.3::real); +ERROR: "attname" cannot be NULL +-- error: attname doesn't exist +SELECT pg_catalog.pg_restore_attribute_stats( + 'relation', 'stats_import.test'::regclass, + 'attname', 'nope'::name, + 'inherited', false::boolean, + 'version', 150000::integer, + 'null_frac', 0.1::real, + 'avg_width', 2::integer, + 'n_distinct', 0.3::real); +ERROR: column "nope" of relation "test" does not exist +-- error: inherited null +SELECT pg_catalog.pg_restore_attribute_stats( + 'relation', 'stats_import.test'::regclass, + 'attname', 'id'::name, + 'inherited', NULL::boolean, + 'version', 150000::integer, + 'null_frac', 0.1::real, + 'avg_width', 2::integer, + 'n_distinct', 0.3::real); +ERROR: "inherited" cannot be NULL +-- ok: no stakinds +SELECT pg_catalog.pg_restore_attribute_stats( + 'relation', 'stats_import.test'::regclass, + 'attname', 'id'::name, + 'inherited', false::boolean, + 'version', 150000::integer, + 'null_frac', 0.4::real, + 'avg_width', 5::integer, + 'n_distinct', 0.6::real); + pg_restore_attribute_stats +---------------------------- + t +(1 row) + +SELECT * +FROM pg_stats +WHERE schemaname = 'stats_import' +AND tablename = 'test' +AND inherited = false +AND attname = 'id'; + schemaname | tablename | attname | inherited | null_frac | avg_width | n_distinct | most_common_vals | most_common_freqs | histogram_bounds | correlation | most_common_elems | most_common_elem_freqs | elem_count_histogram | range_length_histogram | range_empty_frac | range_bounds_histogram +--------------+-----------+---------+-----------+-----------+-----------+------------+------------------+-------------------+------------------+-------------+-------------------+------------------------+----------------------+------------------------+------------------+------------------------ + stats_import | test | id | f | 0.4 | 5 | 0.6 | | | | | | | | | | +(1 row) + +-- warn: unrecognized argument name +SELECT pg_catalog.pg_restore_attribute_stats( + 'relation', 'stats_import.test'::regclass, + 'attname', 'id'::name, + 'inherited', false::boolean, + 'version', 150000::integer, + 'null_frac', 0.2::real, + 'avg_width', NULL::integer, + 'nope', 0.5::real); +WARNING: unrecognized argument name: "nope" + pg_restore_attribute_stats +---------------------------- + f +(1 row) + +SELECT * +FROM pg_stats +WHERE schemaname = 'stats_import' +AND tablename = 'test' +AND inherited = false +AND attname = 'id'; + schemaname | tablename | attname | inherited | null_frac | avg_width | n_distinct | most_common_vals | most_common_freqs | histogram_bounds | correlation | most_common_elems | most_common_elem_freqs | elem_count_histogram | range_length_histogram | range_empty_frac | range_bounds_histogram +--------------+-----------+---------+-----------+-----------+-----------+------------+------------------+-------------------+------------------+-------------+-------------------+------------------------+----------------------+------------------------+------------------+------------------------ + stats_import | test | id | f | 0.2 | 5 | 0.6 | | | | | | | | | | +(1 row) + +-- warn: mcv / mcf null mismatch part 1 +SELECT pg_catalog.pg_restore_attribute_stats( + 'relation', 'stats_import.test'::regclass, + 'attname', 'id'::name, + 'inherited', false::boolean, + 'version', 150000::integer, + 'null_frac', 0.6::real, + 'avg_width', 7::integer, + 'n_distinct', -0.7::real, + 'most_common_freqs', '{0.1,0.2,0.3}'::real[] + ); +WARNING: "most_common_vals" must be specified when "most_common_freqs" is specified + pg_restore_attribute_stats +---------------------------- + f +(1 row) + +SELECT * +FROM pg_stats +WHERE schemaname = 'stats_import' +AND tablename = 'test' +AND inherited = false +AND attname = 'id'; + schemaname | tablename | attname | inherited | null_frac | avg_width | n_distinct | most_common_vals | most_common_freqs | histogram_bounds | correlation | most_common_elems | most_common_elem_freqs | elem_count_histogram | range_length_histogram | range_empty_frac | range_bounds_histogram +--------------+-----------+---------+-----------+-----------+-----------+------------+------------------+-------------------+------------------+-------------+-------------------+------------------------+----------------------+------------------------+------------------+------------------------ + stats_import | test | id | f | 0.6 | 7 | -0.7 | | | | | | | | | | +(1 row) + +-- warn: mcv / mcf null mismatch part 2 +SELECT pg_catalog.pg_restore_attribute_stats( + 'relation', 'stats_import.test'::regclass, + 'attname', 'id'::name, + 'inherited', false::boolean, + 'version', 150000::integer, + 'null_frac', 0.7::real, + 'avg_width', 8::integer, + 'n_distinct', -0.8::real, + 'most_common_vals', '{1,2,3}'::text + ); +WARNING: "most_common_freqs" must be specified when "most_common_vals" is specified + pg_restore_attribute_stats +---------------------------- + f +(1 row) + +SELECT * +FROM pg_stats +WHERE schemaname = 'stats_import' +AND tablename = 'test' +AND inherited = false +AND attname = 'id'; + schemaname | tablename | attname | inherited | null_frac | avg_width | n_distinct | most_common_vals | most_common_freqs | histogram_bounds | correlation | most_common_elems | most_common_elem_freqs | elem_count_histogram | range_length_histogram | range_empty_frac | range_bounds_histogram +--------------+-----------+---------+-----------+-----------+-----------+------------+------------------+-------------------+------------------+-------------+-------------------+------------------------+----------------------+------------------------+------------------+------------------------ + stats_import | test | id | f | 0.7 | 8 | -0.8 | | | | | | | | | | +(1 row) + +-- warn: mcv / mcf type mismatch +SELECT pg_catalog.pg_restore_attribute_stats( + 'relation', 'stats_import.test'::regclass, + 'attname', 'id'::name, + 'inherited', false::boolean, + 'version', 150000::integer, + 'null_frac', 0.8::real, + 'avg_width', 9::integer, + 'n_distinct', -0.9::real, + 'most_common_vals', '{2,1,3}'::text, + 'most_common_freqs', '{0.2,0.1}'::double precision[] + ); +WARNING: argument "most_common_freqs" has type "double precision[]", expected type "real[]" +WARNING: "most_common_freqs" must be specified when "most_common_vals" is specified + pg_restore_attribute_stats +---------------------------- + f +(1 row) + +SELECT * +FROM pg_stats +WHERE schemaname = 'stats_import' +AND tablename = 'test' +AND inherited = false +AND attname = 'id'; + schemaname | tablename | attname | inherited | null_frac | avg_width | n_distinct | most_common_vals | most_common_freqs | histogram_bounds | correlation | most_common_elems | most_common_elem_freqs | elem_count_histogram | range_length_histogram | range_empty_frac | range_bounds_histogram +--------------+-----------+---------+-----------+-----------+-----------+------------+------------------+-------------------+------------------+-------------+-------------------+------------------------+----------------------+------------------------+------------------+------------------------ + stats_import | test | id | f | 0.8 | 9 | -0.9 | | | | | | | | | | +(1 row) + +-- warn: mcv cast failure +SELECT pg_catalog.pg_restore_attribute_stats( + 'relation', 'stats_import.test'::regclass, + 'attname', 'id'::name, + 'inherited', false::boolean, + 'version', 150000::integer, + 'null_frac', 0.9::real, + 'avg_width', 10::integer, + 'n_distinct', -0.4::real, + 'most_common_vals', '{2,four,3}'::text, + 'most_common_freqs', '{0.3,0.25,0.05}'::real[] + ); +WARNING: invalid input syntax for type integer: "four" + pg_restore_attribute_stats +---------------------------- + f +(1 row) + +SELECT * +FROM pg_stats +WHERE schemaname = 'stats_import' +AND tablename = 'test' +AND inherited = false +AND attname = 'id'; + schemaname | tablename | attname | inherited | null_frac | avg_width | n_distinct | most_common_vals | most_common_freqs | histogram_bounds | correlation | most_common_elems | most_common_elem_freqs | elem_count_histogram | range_length_histogram | range_empty_frac | range_bounds_histogram +--------------+-----------+---------+-----------+-----------+-----------+------------+------------------+-------------------+------------------+-------------+-------------------+------------------------+----------------------+------------------------+------------------+------------------------ + stats_import | test | id | f | 0.9 | 10 | -0.4 | | | | | | | | | | +(1 row) + +-- ok: mcv+mcf +SELECT pg_catalog.pg_restore_attribute_stats( + 'relation', 'stats_import.test'::regclass, + 'attname', 'id'::name, + 'inherited', false::boolean, + 'version', 150000::integer, + 'null_frac', 0.1::real, + 'avg_width', 1::integer, + 'n_distinct', -0.1::real, + 'most_common_vals', '{2,1,3}'::text, + 'most_common_freqs', '{0.3,0.25,0.05}'::real[] + ); + pg_restore_attribute_stats +---------------------------- + t +(1 row) + +SELECT * +FROM pg_stats +WHERE schemaname = 'stats_import' +AND tablename = 'test' +AND inherited = false +AND attname = 'id'; + schemaname | tablename | attname | inherited | null_frac | avg_width | n_distinct | most_common_vals | most_common_freqs | histogram_bounds | correlation | most_common_elems | most_common_elem_freqs | elem_count_histogram | range_length_histogram | range_empty_frac | range_bounds_histogram +--------------+-----------+---------+-----------+-----------+-----------+------------+------------------+-------------------+------------------+-------------+-------------------+------------------------+----------------------+------------------------+------------------+------------------------ + stats_import | test | id | f | 0.1 | 1 | -0.1 | {2,1,3} | {0.3,0.25,0.05} | | | | | | | | +(1 row) + +-- warn: NULL in histogram array +SELECT pg_catalog.pg_restore_attribute_stats( + 'relation', 'stats_import.test'::regclass, + 'attname', 'id'::name, + 'inherited', false::boolean, + 'version', 150000::integer, + 'null_frac', 0.2::real, + 'avg_width', 2::integer, + 'n_distinct', -0.2::real, + 'histogram_bounds', '{1,NULL,3,4}'::text + ); +WARNING: "histogram_bounds" array cannot contain NULL values + pg_restore_attribute_stats +---------------------------- + f +(1 row) + +SELECT * +FROM pg_stats +WHERE schemaname = 'stats_import' +AND tablename = 'test' +AND inherited = false +AND attname = 'id'; + schemaname | tablename | attname | inherited | null_frac | avg_width | n_distinct | most_common_vals | most_common_freqs | histogram_bounds | correlation | most_common_elems | most_common_elem_freqs | elem_count_histogram | range_length_histogram | range_empty_frac | range_bounds_histogram +--------------+-----------+---------+-----------+-----------+-----------+------------+------------------+-------------------+------------------+-------------+-------------------+------------------------+----------------------+------------------------+------------------+------------------------ + stats_import | test | id | f | 0.2 | 2 | -0.2 | {2,1,3} | {0.3,0.25,0.05} | | | | | | | | +(1 row) + +-- ok: histogram_bounds +SELECT pg_catalog.pg_restore_attribute_stats( + 'relation', 'stats_import.test'::regclass, + 'attname', 'id'::name, + 'inherited', false::boolean, + 'version', 150000::integer, + 'null_frac', 0.3::real, + 'avg_width', 3::integer, + 'n_distinct', -0.3::real, + 'histogram_bounds', '{1,2,3,4}'::text ); + pg_restore_attribute_stats +---------------------------- + t +(1 row) + +SELECT * +FROM pg_stats +WHERE schemaname = 'stats_import' +AND tablename = 'test' +AND inherited = false +AND attname = 'id'; + schemaname | tablename | attname | inherited | null_frac | avg_width | n_distinct | most_common_vals | most_common_freqs | histogram_bounds | correlation | most_common_elems | most_common_elem_freqs | elem_count_histogram | range_length_histogram | range_empty_frac | range_bounds_histogram +--------------+-----------+---------+-----------+-----------+-----------+------------+------------------+-------------------+------------------+-------------+-------------------+------------------------+----------------------+------------------------+------------------+------------------------ + stats_import | test | id | f | 0.3 | 3 | -0.3 | {2,1,3} | {0.3,0.25,0.05} | {1,2,3,4} | | | | | | | +(1 row) + +-- warn: elem_count_histogram null element +SELECT pg_catalog.pg_restore_attribute_stats( + 'relation', 'stats_import.test'::regclass, + 'attname', 'tags'::name, + 'inherited', false::boolean, + 'version', 150000::integer, + 'null_frac', 0.4::real, + 'avg_width', 5::integer, + 'n_distinct', -0.4::real, + 'elem_count_histogram', '{1,1,NULL,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}'::real[] + ); +WARNING: "elem_count_histogram" array cannot contain NULL values + pg_restore_attribute_stats +---------------------------- + f +(1 row) + +SELECT * +FROM pg_stats +WHERE schemaname = 'stats_import' +AND tablename = 'test' +AND inherited = false +AND attname = 'tags'; + schemaname | tablename | attname | inherited | null_frac | avg_width | n_distinct | most_common_vals | most_common_freqs | histogram_bounds | correlation | most_common_elems | most_common_elem_freqs | elem_count_histogram | range_length_histogram | range_empty_frac | range_bounds_histogram +--------------+-----------+---------+-----------+-----------+-----------+------------+------------------+-------------------+------------------+-------------+-------------------+------------------------+----------------------+------------------------+------------------+------------------------ + stats_import | test | tags | f | 0.4 | 5 | -0.4 | | | | | | | | | | +(1 row) + +-- ok: elem_count_histogram +SELECT pg_catalog.pg_restore_attribute_stats( + 'relation', 'stats_import.test'::regclass, + 'attname', 'tags'::name, + 'inherited', false::boolean, + 'version', 150000::integer, + 'null_frac', 0.5::real, + 'avg_width', 6::integer, + 'n_distinct', -0.55::real, + 'elem_count_histogram', '{1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}'::real[] + ); + pg_restore_attribute_stats +---------------------------- + t +(1 row) + +SELECT * +FROM pg_stats +WHERE schemaname = 'stats_import' +AND tablename = 'test' +AND inherited = false +AND attname = 'tags'; + schemaname | tablename | attname | inherited | null_frac | avg_width | n_distinct | most_common_vals | most_common_freqs | histogram_bounds | correlation | most_common_elems | most_common_elem_freqs | elem_count_histogram | range_length_histogram | range_empty_frac | range_bounds_histogram +--------------+-----------+---------+-----------+-----------+-----------+------------+------------------+-------------------+------------------+-------------+-------------------+------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------+------------------+------------------------ + stats_import | test | tags | f | 0.5 | 6 | -0.55 | | | | | | | {1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1} | | | +(1 row) + +-- range stats on a scalar type +SELECT pg_catalog.pg_restore_attribute_stats( + 'relation', 'stats_import.test'::regclass, + 'attname', 'id'::name, + 'inherited', false::boolean, + 'version', 150000::integer, + 'null_frac', 0.6::real, + 'avg_width', 7::integer, + 'n_distinct', -0.15::real, + 'range_empty_frac', 0.5::real, + 'range_length_histogram', '{399,499,Infinity}'::text + ); +WARNING: attribute "id" is not a range type +DETAIL: Cannot set STATISTIC_KIND_RANGE_LENGTH_HISTOGRAM or STATISTIC_KIND_BOUNDS_HISTOGRAM. + pg_restore_attribute_stats +---------------------------- + f +(1 row) + +SELECT * +FROM pg_stats +WHERE schemaname = 'stats_import' +AND tablename = 'test' +AND inherited = false +AND attname = 'id'; + schemaname | tablename | attname | inherited | null_frac | avg_width | n_distinct | most_common_vals | most_common_freqs | histogram_bounds | correlation | most_common_elems | most_common_elem_freqs | elem_count_histogram | range_length_histogram | range_empty_frac | range_bounds_histogram +--------------+-----------+---------+-----------+-----------+-----------+------------+------------------+-------------------+------------------+-------------+-------------------+------------------------+----------------------+------------------------+------------------+------------------------ + stats_import | test | id | f | 0.6 | 7 | -0.15 | {2,1,3} | {0.3,0.25,0.05} | {1,2,3,4} | | | | | | | +(1 row) + +-- warn: range_empty_frac range_length_hist null mismatch +SELECT pg_catalog.pg_restore_attribute_stats( + 'relation', 'stats_import.test'::regclass, + 'attname', 'arange'::name, + 'inherited', false::boolean, + 'version', 150000::integer, + 'null_frac', 0.7::real, + 'avg_width', 8::integer, + 'n_distinct', -0.25::real, + 'range_length_histogram', '{399,499,Infinity}'::text + ); +WARNING: "range_empty_frac" must be specified when "range_length_histogram" is specified + pg_restore_attribute_stats +---------------------------- + f +(1 row) + +SELECT * +FROM pg_stats +WHERE schemaname = 'stats_import' +AND tablename = 'test' +AND inherited = false +AND attname = 'arange'; + schemaname | tablename | attname | inherited | null_frac | avg_width | n_distinct | most_common_vals | most_common_freqs | histogram_bounds | correlation | most_common_elems | most_common_elem_freqs | elem_count_histogram | range_length_histogram | range_empty_frac | range_bounds_histogram +--------------+-----------+---------+-----------+-----------+-----------+------------+------------------+-------------------+------------------+-------------+-------------------+------------------------+----------------------+------------------------+------------------+------------------------ + stats_import | test | arange | f | 0.7 | 8 | -0.25 | | | | | | | | | | +(1 row) + +-- warn: range_empty_frac range_length_hist null mismatch part 2 +SELECT pg_catalog.pg_restore_attribute_stats( + 'relation', 'stats_import.test'::regclass, + 'attname', 'arange'::name, + 'inherited', false::boolean, + 'version', 150000::integer, + 'null_frac', 0.8::real, + 'avg_width', 9::integer, + 'n_distinct', -0.35::real, + 'range_empty_frac', 0.5::real + ); +WARNING: "range_length_histogram" must be specified when "range_empty_frac" is specified + pg_restore_attribute_stats +---------------------------- + f +(1 row) + +SELECT * +FROM pg_stats +WHERE schemaname = 'stats_import' +AND tablename = 'test' +AND inherited = false +AND attname = 'arange'; + schemaname | tablename | attname | inherited | null_frac | avg_width | n_distinct | most_common_vals | most_common_freqs | histogram_bounds | correlation | most_common_elems | most_common_elem_freqs | elem_count_histogram | range_length_histogram | range_empty_frac | range_bounds_histogram +--------------+-----------+---------+-----------+-----------+-----------+------------+------------------+-------------------+------------------+-------------+-------------------+------------------------+----------------------+------------------------+------------------+------------------------ + stats_import | test | arange | f | 0.8 | 9 | -0.35 | | | | | | | | | | +(1 row) + +-- ok: range_empty_frac + range_length_hist +SELECT pg_catalog.pg_restore_attribute_stats( + 'relation', 'stats_import.test'::regclass, + 'attname', 'arange'::name, + 'inherited', false::boolean, + 'version', 150000::integer, + 'null_frac', 0.9::real, + 'avg_width', 1::integer, + 'n_distinct', -0.19::real, + 'range_empty_frac', 0.5::real, + 'range_length_histogram', '{399,499,Infinity}'::text + ); + pg_restore_attribute_stats +---------------------------- + t +(1 row) + +SELECT * +FROM pg_stats +WHERE schemaname = 'stats_import' +AND tablename = 'test' +AND inherited = false +AND attname = 'arange'; + schemaname | tablename | attname | inherited | null_frac | avg_width | n_distinct | most_common_vals | most_common_freqs | histogram_bounds | correlation | most_common_elems | most_common_elem_freqs | elem_count_histogram | range_length_histogram | range_empty_frac | range_bounds_histogram +--------------+-----------+---------+-----------+-----------+-----------+------------+------------------+-------------------+------------------+-------------+-------------------+------------------------+----------------------+------------------------+------------------+------------------------ + stats_import | test | arange | f | 0.9 | 1 | -0.19 | | | | | | | | {399,499,Infinity} | 0.5 | +(1 row) + +-- warn: range bounds histogram on scalar +SELECT pg_catalog.pg_restore_attribute_stats( + 'relation', 'stats_import.test'::regclass, + 'attname', 'id'::name, + 'inherited', false::boolean, + 'version', 150000::integer, + 'null_frac', 0.1::real, + 'avg_width', 2::integer, + 'n_distinct', -0.29::real, + 'range_bounds_histogram', '{"[-1,1)","[0,4)","[1,4)","[1,100)"}'::text + ); +WARNING: attribute "id" is not a range type +DETAIL: Cannot set STATISTIC_KIND_RANGE_LENGTH_HISTOGRAM or STATISTIC_KIND_BOUNDS_HISTOGRAM. + pg_restore_attribute_stats +---------------------------- + f +(1 row) + +SELECT * +FROM pg_stats +WHERE schemaname = 'stats_import' +AND tablename = 'test' +AND inherited = false +AND attname = 'id'; + schemaname | tablename | attname | inherited | null_frac | avg_width | n_distinct | most_common_vals | most_common_freqs | histogram_bounds | correlation | most_common_elems | most_common_elem_freqs | elem_count_histogram | range_length_histogram | range_empty_frac | range_bounds_histogram +--------------+-----------+---------+-----------+-----------+-----------+------------+------------------+-------------------+------------------+-------------+-------------------+------------------------+----------------------+------------------------+------------------+------------------------ + stats_import | test | id | f | 0.1 | 2 | -0.29 | {2,1,3} | {0.3,0.25,0.05} | {1,2,3,4} | | | | | | | +(1 row) + +-- ok: range_bounds_histogram +SELECT pg_catalog.pg_restore_attribute_stats( + 'relation', 'stats_import.test'::regclass, + 'attname', 'arange'::name, + 'inherited', false::boolean, + 'version', 150000::integer, + 'null_frac', 0.2::real, + 'avg_width', 3::integer, + 'n_distinct', -0.39::real, + 'range_bounds_histogram', '{"[-1,1)","[0,4)","[1,4)","[1,100)"}'::text + ); + pg_restore_attribute_stats +---------------------------- + t +(1 row) + +SELECT * +FROM pg_stats +WHERE schemaname = 'stats_import' +AND tablename = 'test' +AND inherited = false +AND attname = 'arange'; + schemaname | tablename | attname | inherited | null_frac | avg_width | n_distinct | most_common_vals | most_common_freqs | histogram_bounds | correlation | most_common_elems | most_common_elem_freqs | elem_count_histogram | range_length_histogram | range_empty_frac | range_bounds_histogram +--------------+-----------+---------+-----------+-----------+-----------+------------+------------------+-------------------+------------------+-------------+-------------------+------------------------+----------------------+------------------------+------------------+-------------------------------------- + stats_import | test | arange | f | 0.2 | 3 | -0.39 | | | | | | | | {399,499,Infinity} | 0.5 | {"[-1,1)","[0,4)","[1,4)","[1,100)"} +(1 row) + +-- warn: too many stat kinds +SELECT pg_catalog.pg_restore_attribute_stats( + 'relation', 'stats_import.test'::regclass, + 'attname', 'arange'::name, + 'inherited', false::boolean, + 'version', 150000::integer, + 'null_frac', 0.5::real, + 'avg_width', 2::integer, + 'n_distinct', -0.1::real, + 'most_common_vals', '{"[2,3)","[1,3)","[3,9)"}'::text, + 'most_common_freqs', '{0.3,0.25,0.05}'::real[], + 'histogram_bounds', '{"[1,2)","[2,3)","[3,4)","[4,)"}'::text, + 'correlation', 1.1::real, + 'most_common_elems', '{3,1}'::text, + 'most_common_elem_freqs', '{0.3,0.2,0.2,0.3,0.0}'::real[], + 'range_empty_frac', -0.5::real, + 'range_length_histogram', '{399,499,Infinity}'::text, + 'range_bounds_histogram', '{"[-1,1)","[0,4)","[1,4)","[1,100)"}'::text); +WARNING: unable to determine element type of attribute "arange" +DETAIL: Cannot set STATISTIC_KIND_MCELEM or STATISTIC_KIND_DECHIST. + pg_restore_attribute_stats +---------------------------- + f +(1 row) + -- -- Test the ability to exactly copy data from one table to an identical table, -- correctly reconstructing the stakind order as well as the staopN and @@ -848,6 +1514,217 @@ WHERE s.starelid = 'stats_import.is_odd'::regclass; ---------+------------+-------------+----------+-------------+----------+----------+----------+----------+----------+--------+--------+--------+--------+--------+----------+----------+----------+----------+----------+-------------+-------------+-------------+-------------+-------------+-----+-----+-----+-----+-----+----------- (0 rows) +-- +SELECT relpages, reltuples, relallvisible +FROM pg_class +WHERE oid = 'stats_import.test'::regclass; + relpages | reltuples | relallvisible +----------+-----------+--------------- + 1 | 4 | 0 +(1 row) + +-- +-- Clear clone stats to try again with pg_restore_attribute_stats +-- +SELECT + pg_catalog.pg_clear_attribute_stats( + ('stats_import.' || s.tablename)::regclass, + s.attname, + s.inherited) +FROM pg_catalog.pg_stats AS s +WHERE s.schemaname = 'stats_import' +AND s.tablename IN ('test_clone', 'is_odd_clone') +ORDER BY s.tablename, s.attname, s.inherited; + pg_clear_attribute_stats +-------------------------- + + + + + + +(6 rows) + +SELECT +SELECT COUNT(*) +FROM pg_catalog.pg_stats AS s +WHERE s.schemaname = 'stats_import' +AND s.tablename IN ('test_clone', 'is_odd_clone'); +ERROR: syntax error at or near "SELECT" +LINE 2: SELECT COUNT(*) + ^ +-- +-- Copy stats from test to test_clone, and is_odd to is_odd_clone +-- +SELECT s.schemaname, s.tablename, s.attname, s.inherited, r.* +FROM pg_catalog.pg_stats AS s +CROSS JOIN LATERAL + pg_catalog.pg_restore_attribute_stats( + 'relation', ('stats_import.' || s.tablename || '_clone')::regclass, + 'attname', s.attname, + 'inherited', s.inherited, + 'version', 150000, + 'null_frac', s.null_frac, + 'avg_width', s.avg_width, + 'n_distinct', s.n_distinct, + 'most_common_vals', s.most_common_vals::text, + 'most_common_freqs', s.most_common_freqs, + 'histogram_bounds', s.histogram_bounds::text, + 'correlation', s.correlation, + 'most_common_elems', s.most_common_elems::text, + 'most_common_elem_freqs', s.most_common_elem_freqs, + 'elem_count_histogram', s.elem_count_histogram, + 'range_bounds_histogram', s.range_bounds_histogram::text, + 'range_empty_frac', s.range_empty_frac, + 'range_length_histogram', s.range_length_histogram::text) AS r +WHERE s.schemaname = 'stats_import' +AND s.tablename IN ('test', 'is_odd') +ORDER BY s.tablename, s.attname, s.inherited; + schemaname | tablename | attname | inherited | r +--------------+-----------+---------+-----------+--- + stats_import | is_odd | expr | f | t + stats_import | test | arange | f | t + stats_import | test | comp | f | t + stats_import | test | id | f | t + stats_import | test | name | f | t + stats_import | test | tags | f | t +(6 rows) + +SELECT c.relname, COUNT(*) AS num_stats +FROM pg_class AS c +JOIN pg_statistic s ON s.starelid = c.oid +WHERE c.relnamespace = 'stats_import'::regnamespace +AND c.relname IN ('test', 'test_clone', 'is_odd', 'is_odd_clone') +GROUP BY c.relname +ORDER BY c.relname; + relname | num_stats +--------------+----------- + is_odd | 1 + is_odd_clone | 1 + test | 5 + test_clone | 5 +(4 rows) + +-- check test minus test_clone +SELECT + a.attname, s.stainherit, s.stanullfrac, s.stawidth, s.stadistinct, + s.stakind1, s.stakind2, s.stakind3, s.stakind4, s.stakind5, + s.staop1, s.staop2, s.staop3, s.staop4, s.staop5, + s.stacoll1, s.stacoll2, s.stacoll3, s.stacoll4, s.stacoll5, + s.stanumbers1, s.stanumbers2, s.stanumbers3, s.stanumbers4, s.stanumbers5, + s.stavalues1::text AS sv1, s.stavalues2::text AS sv2, + s.stavalues3::text AS sv3, s.stavalues4::text AS sv4, + s.stavalues5::text AS sv5, 'test' AS direction +FROM pg_statistic s +JOIN pg_attribute a ON a.attrelid = s.starelid AND a.attnum = s.staattnum +WHERE s.starelid = 'stats_import.test'::regclass +EXCEPT +SELECT + a.attname, s.stainherit, s.stanullfrac, s.stawidth, s.stadistinct, + s.stakind1, s.stakind2, s.stakind3, s.stakind4, s.stakind5, + s.staop1, s.staop2, s.staop3, s.staop4, s.staop5, + s.stacoll1, s.stacoll2, s.stacoll3, s.stacoll4, s.stacoll5, + s.stanumbers1, s.stanumbers2, s.stanumbers3, s.stanumbers4, s.stanumbers5, + s.stavalues1::text AS sv1, s.stavalues2::text AS sv2, + s.stavalues3::text AS sv3, s.stavalues4::text AS sv4, + s.stavalues5::text AS sv5, 'test' AS direction +FROM pg_statistic s +JOIN pg_attribute a ON a.attrelid = s.starelid AND a.attnum = s.staattnum +WHERE s.starelid = 'stats_import.test_clone'::regclass; + attname | stainherit | stanullfrac | stawidth | stadistinct | stakind1 | stakind2 | stakind3 | stakind4 | stakind5 | staop1 | staop2 | staop3 | staop4 | staop5 | stacoll1 | stacoll2 | stacoll3 | stacoll4 | stacoll5 | stanumbers1 | stanumbers2 | stanumbers3 | stanumbers4 | stanumbers5 | sv1 | sv2 | sv3 | sv4 | sv5 | direction +---------+------------+-------------+----------+-------------+----------+----------+----------+----------+----------+--------+--------+--------+--------+--------+----------+----------+----------+----------+----------+-------------+-------------+-------------+-------------+-------------+-----+-----+-----+-----+-----+----------- +(0 rows) + +-- check test_clone minus test +SELECT + a.attname, s.stainherit, s.stanullfrac, s.stawidth, s.stadistinct, + s.stakind1, s.stakind2, s.stakind3, s.stakind4, s.stakind5, + s.staop1, s.staop2, s.staop3, s.staop4, s.staop5, + s.stacoll1, s.stacoll2, s.stacoll3, s.stacoll4, s.stacoll5, + s.stanumbers1, s.stanumbers2, s.stanumbers3, s.stanumbers4, s.stanumbers5, + s.stavalues1::text AS sv1, s.stavalues2::text AS sv2, + s.stavalues3::text AS sv3, s.stavalues4::text AS sv4, + s.stavalues5::text AS sv5, 'test_clone' AS direction +FROM pg_statistic s +JOIN pg_attribute a ON a.attrelid = s.starelid AND a.attnum = s.staattnum +WHERE s.starelid = 'stats_import.test_clone'::regclass +EXCEPT +SELECT + a.attname, s.stainherit, s.stanullfrac, s.stawidth, s.stadistinct, + s.stakind1, s.stakind2, s.stakind3, s.stakind4, s.stakind5, + s.staop1, s.staop2, s.staop3, s.staop4, s.staop5, + s.stacoll1, s.stacoll2, s.stacoll3, s.stacoll4, s.stacoll5, + s.stanumbers1, s.stanumbers2, s.stanumbers3, s.stanumbers4, s.stanumbers5, + s.stavalues1::text AS sv1, s.stavalues2::text AS sv2, + s.stavalues3::text AS sv3, s.stavalues4::text AS sv4, + s.stavalues5::text AS sv5, 'test_clone' AS direction +FROM pg_statistic s +JOIN pg_attribute a ON a.attrelid = s.starelid AND a.attnum = s.staattnum +WHERE s.starelid = 'stats_import.test'::regclass; + attname | stainherit | stanullfrac | stawidth | stadistinct | stakind1 | stakind2 | stakind3 | stakind4 | stakind5 | staop1 | staop2 | staop3 | staop4 | staop5 | stacoll1 | stacoll2 | stacoll3 | stacoll4 | stacoll5 | stanumbers1 | stanumbers2 | stanumbers3 | stanumbers4 | stanumbers5 | sv1 | sv2 | sv3 | sv4 | sv5 | direction +---------+------------+-------------+----------+-------------+----------+----------+----------+----------+----------+--------+--------+--------+--------+--------+----------+----------+----------+----------+----------+-------------+-------------+-------------+-------------+-------------+-----+-----+-----+-----+-----+----------- +(0 rows) + +-- check is_odd minus is_odd_clone +SELECT + a.attname, s.stainherit, s.stanullfrac, s.stawidth, s.stadistinct, + s.stakind1, s.stakind2, s.stakind3, s.stakind4, s.stakind5, + s.staop1, s.staop2, s.staop3, s.staop4, s.staop5, + s.stacoll1, s.stacoll2, s.stacoll3, s.stacoll4, s.stacoll5, + s.stanumbers1, s.stanumbers2, s.stanumbers3, s.stanumbers4, s.stanumbers5, + s.stavalues1::text AS sv1, s.stavalues2::text AS sv2, + s.stavalues3::text AS sv3, s.stavalues4::text AS sv4, + s.stavalues5::text AS sv5, 'is_odd' AS direction +FROM pg_statistic s +JOIN pg_attribute a ON a.attrelid = s.starelid AND a.attnum = s.staattnum +WHERE s.starelid = 'stats_import.is_odd'::regclass +EXCEPT +SELECT + a.attname, s.stainherit, s.stanullfrac, s.stawidth, s.stadistinct, + s.stakind1, s.stakind2, s.stakind3, s.stakind4, s.stakind5, + s.staop1, s.staop2, s.staop3, s.staop4, s.staop5, + s.stacoll1, s.stacoll2, s.stacoll3, s.stacoll4, s.stacoll5, + s.stanumbers1, s.stanumbers2, s.stanumbers3, s.stanumbers4, s.stanumbers5, + s.stavalues1::text AS sv1, s.stavalues2::text AS sv2, + s.stavalues3::text AS sv3, s.stavalues4::text AS sv4, + s.stavalues5::text AS sv5, 'is_odd' AS direction +FROM pg_statistic s +JOIN pg_attribute a ON a.attrelid = s.starelid AND a.attnum = s.staattnum +WHERE s.starelid = 'stats_import.is_odd_clone'::regclass; + attname | stainherit | stanullfrac | stawidth | stadistinct | stakind1 | stakind2 | stakind3 | stakind4 | stakind5 | staop1 | staop2 | staop3 | staop4 | staop5 | stacoll1 | stacoll2 | stacoll3 | stacoll4 | stacoll5 | stanumbers1 | stanumbers2 | stanumbers3 | stanumbers4 | stanumbers5 | sv1 | sv2 | sv3 | sv4 | sv5 | direction +---------+------------+-------------+----------+-------------+----------+----------+----------+----------+----------+--------+--------+--------+--------+--------+----------+----------+----------+----------+----------+-------------+-------------+-------------+-------------+-------------+-----+-----+-----+-----+-----+----------- +(0 rows) + +-- check is_odd_clone minus is_odd +SELECT + a.attname, s.stainherit, s.stanullfrac, s.stawidth, s.stadistinct, + s.stakind1, s.stakind2, s.stakind3, s.stakind4, s.stakind5, + s.staop1, s.staop2, s.staop3, s.staop4, s.staop5, + s.stacoll1, s.stacoll2, s.stacoll3, s.stacoll4, s.stacoll5, + s.stanumbers1, s.stanumbers2, s.stanumbers3, s.stanumbers4, s.stanumbers5, + s.stavalues1::text AS sv1, s.stavalues2::text AS sv2, + s.stavalues3::text AS sv3, s.stavalues4::text AS sv4, + s.stavalues5::text AS sv5, 'is_odd_clone' AS direction +FROM pg_statistic s +JOIN pg_attribute a ON a.attrelid = s.starelid AND a.attnum = s.staattnum +WHERE s.starelid = 'stats_import.is_odd_clone'::regclass +EXCEPT +SELECT + a.attname, s.stainherit, s.stanullfrac, s.stawidth, s.stadistinct, + s.stakind1, s.stakind2, s.stakind3, s.stakind4, s.stakind5, + s.staop1, s.staop2, s.staop3, s.staop4, s.staop5, + s.stacoll1, s.stacoll2, s.stacoll3, s.stacoll4, s.stacoll5, + s.stanumbers1, s.stanumbers2, s.stanumbers3, s.stanumbers4, s.stanumbers5, + s.stavalues1::text AS sv1, s.stavalues2::text AS sv2, + s.stavalues3::text AS sv3, s.stavalues4::text AS sv4, + s.stavalues5::text AS sv5, 'is_odd_clone' AS direction +FROM pg_statistic s +JOIN pg_attribute a ON a.attrelid = s.starelid AND a.attnum = s.staattnum +WHERE s.starelid = 'stats_import.is_odd'::regclass; + attname | stainherit | stanullfrac | stawidth | stadistinct | stakind1 | stakind2 | stakind3 | stakind4 | stakind5 | staop1 | staop2 | staop3 | staop4 | staop5 | stacoll1 | stacoll2 | stacoll3 | stacoll4 | stacoll5 | stanumbers1 | stanumbers2 | stanumbers3 | stanumbers4 | stanumbers5 | sv1 | sv2 | sv3 | sv4 | sv5 | direction +---------+------------+-------------+----------+-------------+----------+----------+----------+----------+----------+--------+--------+--------+--------+--------+----------+----------+----------+----------+----------+-------------+-------------+-------------+-------------+-------------+-----+-----+-----+-----+-----+----------- +(0 rows) + DROP SCHEMA stats_import CASCADE; NOTICE: drop cascades to 6 other objects DETAIL: drop cascades to type stats_import.complex_type diff --git a/src/test/regress/sql/stats_import.sql b/src/test/regress/sql/stats_import.sql index b5689ac8fb3..c7d5e017d90 100644 --- a/src/test/regress/sql/stats_import.sql +++ b/src/test/regress/sql/stats_import.sql @@ -178,33 +178,6 @@ SELECT pg_catalog.pg_set_attribute_stats( avg_width => 2::integer, n_distinct => 0.3::real); --- error: null_frac null -SELECT pg_catalog.pg_set_attribute_stats( - relation => 'stats_import.test'::regclass, - attname => 'id'::name, - inherited => false::boolean, - null_frac => NULL::real, - avg_width => 2::integer, - n_distinct => 0.3::real); - --- error: avg_width null -SELECT pg_catalog.pg_set_attribute_stats( - relation => 'stats_import.test'::regclass, - attname => 'id'::name, - inherited => false::boolean, - null_frac => 0.1::real, - avg_width => NULL::integer, - n_distinct => 0.3::real); - --- error: avg_width null -SELECT pg_catalog.pg_set_attribute_stats( - relation => 'stats_import.test'::regclass, - attname => 'id'::name, - inherited => false::boolean, - null_frac => 0.1::real, - avg_width => 2::integer, - n_distinct => NULL::real); - -- ok: no stakinds SELECT pg_catalog.pg_set_attribute_stats( relation => 'stats_import.test'::regclass, @@ -517,6 +490,507 @@ SELECT pg_catalog.pg_set_attribute_stats( range_length_histogram => '{399,499,Infinity}'::text, range_bounds_histogram => '{"[-1,1)","[0,4)","[1,4)","[1,100)"}'::text ); + +-- +-- Clear attribute stats to try again with restore functions +-- (relation stats were already cleared). +-- +SELECT + pg_catalog.pg_clear_attribute_stats( + 'stats_import.test'::regclass, + s.attname, + s.inherited) +FROM pg_catalog.pg_stats AS s +WHERE s.schemaname = 'stats_import' +AND s.tablename = 'test' +ORDER BY s.attname, s.inherited; + +-- reject: argument name is NULL +SELECT pg_restore_relation_stats( + 'relation', '0'::oid::regclass, + 'version', 150000::integer, + NULL, '17'::integer, + 'reltuples', 400::real, + 'relallvisible', 4::integer); + +-- reject: argument name is an integer +SELECT pg_restore_relation_stats( + 'relation', '0'::oid::regclass, + 'version', 150000::integer, + 17, '17'::integer, + 'reltuples', 400::real, + 'relallvisible', 4::integer); + +-- reject: odd number of variadic arguments cannot be pairs +SELECT pg_restore_relation_stats( + 'relation', '0'::oid::regclass, + 'version', 150000::integer, + 'relpages', '17'::integer, + 'reltuples', 400::real, + 'relallvisible'); + +-- reject: object doesn't exist +SELECT pg_restore_relation_stats( + 'relation', '0'::oid::regclass, + 'version', 150000::integer, + 'relpages', '17'::integer, + 'reltuples', 400::real, + 'relallvisible', 4::integer); + +-- ok: set all stats +SELECT pg_restore_relation_stats( + 'relation', 'stats_import.test'::regclass, + 'version', 150000::integer, + 'relpages', '17'::integer, + 'reltuples', 400::real, + 'relallvisible', 4::integer); + +SELECT relpages, reltuples, relallvisible +FROM pg_class +WHERE oid = 'stats_import.test'::regclass; + +-- ok: just relpages +SELECT pg_restore_relation_stats( + 'relation', 'stats_import.test'::regclass, + 'version', 150000::integer, + 'relpages', '16'::integer); + +SELECT relpages, reltuples, relallvisible +FROM pg_class +WHERE oid = 'stats_import.test'::regclass; + +-- ok: just reltuples +SELECT pg_restore_relation_stats( + 'relation', 'stats_import.test'::regclass, + 'version', 150000::integer, + 'reltuples', '500'::real); + +SELECT relpages, reltuples, relallvisible +FROM pg_class +WHERE oid = 'stats_import.test'::regclass; + +-- ok: just relallvisible +SELECT pg_restore_relation_stats( + 'relation', 'stats_import.test'::regclass, + 'version', 150000::integer, + 'relallvisible', 5::integer); + +SELECT relpages, reltuples, relallvisible +FROM pg_class +WHERE oid = 'stats_import.test'::regclass; + +-- warn and error: unrecognized argument name +SELECT pg_restore_relation_stats( + 'relation', '0'::oid::regclass, + 'version', 150000::integer, + 'relpages', '17'::integer, + 'reltuples', 400::real, + 'nope', 4::integer); + +-- warn: bad relpages type +SELECT pg_restore_relation_stats( + 'relation', 'stats_import.test'::regclass, + 'version', 150000::integer, + 'relpages', 'nope'::text, + 'reltuples', 400.0::real, + 'relallvisible', 4::integer); + +SELECT relpages, reltuples, relallvisible +FROM pg_class +WHERE oid = 'stats_import.test'::regclass; + +-- error: object does not exist +SELECT pg_catalog.pg_restore_attribute_stats( + 'relation', '0'::oid::regclass, + 'attname', 'id'::name, + 'inherited', false::boolean, + 'version', 150000::integer, + 'null_frac', 0.1::real, + 'avg_width', 2::integer, + 'n_distinct', 0.3::real); + +-- error: relation null +SELECT pg_catalog.pg_restore_attribute_stats( + 'relation', NULL::oid, + 'attname', 'id'::name, + 'inherited', false::boolean, + 'version', 150000::integer, + 'null_frac', 0.1::real, + 'avg_width', 2::integer, + 'n_distinct', 0.3::real); + +-- error: attname null +SELECT pg_catalog.pg_restore_attribute_stats( + 'relation', 'stats_import.test'::regclass, + 'attname', NULL::name, + 'inherited', false::boolean, + 'version', 150000::integer, + 'null_frac', 0.1::real, + 'avg_width', 2::integer, + 'n_distinct', 0.3::real); + +-- error: attname doesn't exist +SELECT pg_catalog.pg_restore_attribute_stats( + 'relation', 'stats_import.test'::regclass, + 'attname', 'nope'::name, + 'inherited', false::boolean, + 'version', 150000::integer, + 'null_frac', 0.1::real, + 'avg_width', 2::integer, + 'n_distinct', 0.3::real); + +-- error: inherited null +SELECT pg_catalog.pg_restore_attribute_stats( + 'relation', 'stats_import.test'::regclass, + 'attname', 'id'::name, + 'inherited', NULL::boolean, + 'version', 150000::integer, + 'null_frac', 0.1::real, + 'avg_width', 2::integer, + 'n_distinct', 0.3::real); + +-- ok: no stakinds +SELECT pg_catalog.pg_restore_attribute_stats( + 'relation', 'stats_import.test'::regclass, + 'attname', 'id'::name, + 'inherited', false::boolean, + 'version', 150000::integer, + 'null_frac', 0.4::real, + 'avg_width', 5::integer, + 'n_distinct', 0.6::real); + +SELECT * +FROM pg_stats +WHERE schemaname = 'stats_import' +AND tablename = 'test' +AND inherited = false +AND attname = 'id'; + +-- warn: unrecognized argument name +SELECT pg_catalog.pg_restore_attribute_stats( + 'relation', 'stats_import.test'::regclass, + 'attname', 'id'::name, + 'inherited', false::boolean, + 'version', 150000::integer, + 'null_frac', 0.2::real, + 'avg_width', NULL::integer, + 'nope', 0.5::real); + +SELECT * +FROM pg_stats +WHERE schemaname = 'stats_import' +AND tablename = 'test' +AND inherited = false +AND attname = 'id'; + +-- warn: mcv / mcf null mismatch part 1 +SELECT pg_catalog.pg_restore_attribute_stats( + 'relation', 'stats_import.test'::regclass, + 'attname', 'id'::name, + 'inherited', false::boolean, + 'version', 150000::integer, + 'null_frac', 0.6::real, + 'avg_width', 7::integer, + 'n_distinct', -0.7::real, + 'most_common_freqs', '{0.1,0.2,0.3}'::real[] + ); + +SELECT * +FROM pg_stats +WHERE schemaname = 'stats_import' +AND tablename = 'test' +AND inherited = false +AND attname = 'id'; + +-- warn: mcv / mcf null mismatch part 2 +SELECT pg_catalog.pg_restore_attribute_stats( + 'relation', 'stats_import.test'::regclass, + 'attname', 'id'::name, + 'inherited', false::boolean, + 'version', 150000::integer, + 'null_frac', 0.7::real, + 'avg_width', 8::integer, + 'n_distinct', -0.8::real, + 'most_common_vals', '{1,2,3}'::text + ); + +SELECT * +FROM pg_stats +WHERE schemaname = 'stats_import' +AND tablename = 'test' +AND inherited = false +AND attname = 'id'; + +-- warn: mcv / mcf type mismatch +SELECT pg_catalog.pg_restore_attribute_stats( + 'relation', 'stats_import.test'::regclass, + 'attname', 'id'::name, + 'inherited', false::boolean, + 'version', 150000::integer, + 'null_frac', 0.8::real, + 'avg_width', 9::integer, + 'n_distinct', -0.9::real, + 'most_common_vals', '{2,1,3}'::text, + 'most_common_freqs', '{0.2,0.1}'::double precision[] + ); + +SELECT * +FROM pg_stats +WHERE schemaname = 'stats_import' +AND tablename = 'test' +AND inherited = false +AND attname = 'id'; + +-- warn: mcv cast failure +SELECT pg_catalog.pg_restore_attribute_stats( + 'relation', 'stats_import.test'::regclass, + 'attname', 'id'::name, + 'inherited', false::boolean, + 'version', 150000::integer, + 'null_frac', 0.9::real, + 'avg_width', 10::integer, + 'n_distinct', -0.4::real, + 'most_common_vals', '{2,four,3}'::text, + 'most_common_freqs', '{0.3,0.25,0.05}'::real[] + ); + +SELECT * +FROM pg_stats +WHERE schemaname = 'stats_import' +AND tablename = 'test' +AND inherited = false +AND attname = 'id'; + +-- ok: mcv+mcf +SELECT pg_catalog.pg_restore_attribute_stats( + 'relation', 'stats_import.test'::regclass, + 'attname', 'id'::name, + 'inherited', false::boolean, + 'version', 150000::integer, + 'null_frac', 0.1::real, + 'avg_width', 1::integer, + 'n_distinct', -0.1::real, + 'most_common_vals', '{2,1,3}'::text, + 'most_common_freqs', '{0.3,0.25,0.05}'::real[] + ); + +SELECT * +FROM pg_stats +WHERE schemaname = 'stats_import' +AND tablename = 'test' +AND inherited = false +AND attname = 'id'; + +-- warn: NULL in histogram array +SELECT pg_catalog.pg_restore_attribute_stats( + 'relation', 'stats_import.test'::regclass, + 'attname', 'id'::name, + 'inherited', false::boolean, + 'version', 150000::integer, + 'null_frac', 0.2::real, + 'avg_width', 2::integer, + 'n_distinct', -0.2::real, + 'histogram_bounds', '{1,NULL,3,4}'::text + ); + +SELECT * +FROM pg_stats +WHERE schemaname = 'stats_import' +AND tablename = 'test' +AND inherited = false +AND attname = 'id'; + +-- ok: histogram_bounds +SELECT pg_catalog.pg_restore_attribute_stats( + 'relation', 'stats_import.test'::regclass, + 'attname', 'id'::name, + 'inherited', false::boolean, + 'version', 150000::integer, + 'null_frac', 0.3::real, + 'avg_width', 3::integer, + 'n_distinct', -0.3::real, + 'histogram_bounds', '{1,2,3,4}'::text ); + +SELECT * +FROM pg_stats +WHERE schemaname = 'stats_import' +AND tablename = 'test' +AND inherited = false +AND attname = 'id'; + +-- warn: elem_count_histogram null element +SELECT pg_catalog.pg_restore_attribute_stats( + 'relation', 'stats_import.test'::regclass, + 'attname', 'tags'::name, + 'inherited', false::boolean, + 'version', 150000::integer, + 'null_frac', 0.4::real, + 'avg_width', 5::integer, + 'n_distinct', -0.4::real, + 'elem_count_histogram', '{1,1,NULL,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}'::real[] + ); + +SELECT * +FROM pg_stats +WHERE schemaname = 'stats_import' +AND tablename = 'test' +AND inherited = false +AND attname = 'tags'; + +-- ok: elem_count_histogram +SELECT pg_catalog.pg_restore_attribute_stats( + 'relation', 'stats_import.test'::regclass, + 'attname', 'tags'::name, + 'inherited', false::boolean, + 'version', 150000::integer, + 'null_frac', 0.5::real, + 'avg_width', 6::integer, + 'n_distinct', -0.55::real, + 'elem_count_histogram', '{1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}'::real[] + ); + +SELECT * +FROM pg_stats +WHERE schemaname = 'stats_import' +AND tablename = 'test' +AND inherited = false +AND attname = 'tags'; + +-- range stats on a scalar type +SELECT pg_catalog.pg_restore_attribute_stats( + 'relation', 'stats_import.test'::regclass, + 'attname', 'id'::name, + 'inherited', false::boolean, + 'version', 150000::integer, + 'null_frac', 0.6::real, + 'avg_width', 7::integer, + 'n_distinct', -0.15::real, + 'range_empty_frac', 0.5::real, + 'range_length_histogram', '{399,499,Infinity}'::text + ); + +SELECT * +FROM pg_stats +WHERE schemaname = 'stats_import' +AND tablename = 'test' +AND inherited = false +AND attname = 'id'; + +-- warn: range_empty_frac range_length_hist null mismatch +SELECT pg_catalog.pg_restore_attribute_stats( + 'relation', 'stats_import.test'::regclass, + 'attname', 'arange'::name, + 'inherited', false::boolean, + 'version', 150000::integer, + 'null_frac', 0.7::real, + 'avg_width', 8::integer, + 'n_distinct', -0.25::real, + 'range_length_histogram', '{399,499,Infinity}'::text + ); + +SELECT * +FROM pg_stats +WHERE schemaname = 'stats_import' +AND tablename = 'test' +AND inherited = false +AND attname = 'arange'; + +-- warn: range_empty_frac range_length_hist null mismatch part 2 +SELECT pg_catalog.pg_restore_attribute_stats( + 'relation', 'stats_import.test'::regclass, + 'attname', 'arange'::name, + 'inherited', false::boolean, + 'version', 150000::integer, + 'null_frac', 0.8::real, + 'avg_width', 9::integer, + 'n_distinct', -0.35::real, + 'range_empty_frac', 0.5::real + ); + +SELECT * +FROM pg_stats +WHERE schemaname = 'stats_import' +AND tablename = 'test' +AND inherited = false +AND attname = 'arange'; + +-- ok: range_empty_frac + range_length_hist +SELECT pg_catalog.pg_restore_attribute_stats( + 'relation', 'stats_import.test'::regclass, + 'attname', 'arange'::name, + 'inherited', false::boolean, + 'version', 150000::integer, + 'null_frac', 0.9::real, + 'avg_width', 1::integer, + 'n_distinct', -0.19::real, + 'range_empty_frac', 0.5::real, + 'range_length_histogram', '{399,499,Infinity}'::text + ); + +SELECT * +FROM pg_stats +WHERE schemaname = 'stats_import' +AND tablename = 'test' +AND inherited = false +AND attname = 'arange'; + +-- warn: range bounds histogram on scalar +SELECT pg_catalog.pg_restore_attribute_stats( + 'relation', 'stats_import.test'::regclass, + 'attname', 'id'::name, + 'inherited', false::boolean, + 'version', 150000::integer, + 'null_frac', 0.1::real, + 'avg_width', 2::integer, + 'n_distinct', -0.29::real, + 'range_bounds_histogram', '{"[-1,1)","[0,4)","[1,4)","[1,100)"}'::text + ); + +SELECT * +FROM pg_stats +WHERE schemaname = 'stats_import' +AND tablename = 'test' +AND inherited = false +AND attname = 'id'; + +-- ok: range_bounds_histogram +SELECT pg_catalog.pg_restore_attribute_stats( + 'relation', 'stats_import.test'::regclass, + 'attname', 'arange'::name, + 'inherited', false::boolean, + 'version', 150000::integer, + 'null_frac', 0.2::real, + 'avg_width', 3::integer, + 'n_distinct', -0.39::real, + 'range_bounds_histogram', '{"[-1,1)","[0,4)","[1,4)","[1,100)"}'::text + ); + +SELECT * +FROM pg_stats +WHERE schemaname = 'stats_import' +AND tablename = 'test' +AND inherited = false +AND attname = 'arange'; + +-- warn: too many stat kinds +SELECT pg_catalog.pg_restore_attribute_stats( + 'relation', 'stats_import.test'::regclass, + 'attname', 'arange'::name, + 'inherited', false::boolean, + 'version', 150000::integer, + 'null_frac', 0.5::real, + 'avg_width', 2::integer, + 'n_distinct', -0.1::real, + 'most_common_vals', '{"[2,3)","[1,3)","[3,9)"}'::text, + 'most_common_freqs', '{0.3,0.25,0.05}'::real[], + 'histogram_bounds', '{"[1,2)","[2,3)","[3,4)","[4,)"}'::text, + 'correlation', 1.1::real, + 'most_common_elems', '{3,1}'::text, + 'most_common_elem_freqs', '{0.3,0.2,0.2,0.3,0.0}'::real[], + 'range_empty_frac', -0.5::real, + 'range_length_histogram', '{399,499,Infinity}'::text, + 'range_bounds_histogram', '{"[-1,1)","[0,4)","[1,4)","[1,100)"}'::text); + -- -- Test the ability to exactly copy data from one table to an identical table, -- correctly reconstructing the stakind order as well as the staopN and @@ -687,4 +1161,172 @@ FROM pg_statistic s JOIN pg_attribute a ON a.attrelid = s.starelid AND a.attnum = s.staattnum WHERE s.starelid = 'stats_import.is_odd'::regclass; +-- +SELECT relpages, reltuples, relallvisible +FROM pg_class +WHERE oid = 'stats_import.test'::regclass; + +-- +-- Clear clone stats to try again with pg_restore_attribute_stats +-- +SELECT + pg_catalog.pg_clear_attribute_stats( + ('stats_import.' || s.tablename)::regclass, + s.attname, + s.inherited) +FROM pg_catalog.pg_stats AS s +WHERE s.schemaname = 'stats_import' +AND s.tablename IN ('test_clone', 'is_odd_clone') +ORDER BY s.tablename, s.attname, s.inherited; +SELECT + +SELECT COUNT(*) +FROM pg_catalog.pg_stats AS s +WHERE s.schemaname = 'stats_import' +AND s.tablename IN ('test_clone', 'is_odd_clone'); + +-- +-- Copy stats from test to test_clone, and is_odd to is_odd_clone +-- +SELECT s.schemaname, s.tablename, s.attname, s.inherited, r.* +FROM pg_catalog.pg_stats AS s +CROSS JOIN LATERAL + pg_catalog.pg_restore_attribute_stats( + 'relation', ('stats_import.' || s.tablename || '_clone')::regclass, + 'attname', s.attname, + 'inherited', s.inherited, + 'version', 150000, + 'null_frac', s.null_frac, + 'avg_width', s.avg_width, + 'n_distinct', s.n_distinct, + 'most_common_vals', s.most_common_vals::text, + 'most_common_freqs', s.most_common_freqs, + 'histogram_bounds', s.histogram_bounds::text, + 'correlation', s.correlation, + 'most_common_elems', s.most_common_elems::text, + 'most_common_elem_freqs', s.most_common_elem_freqs, + 'elem_count_histogram', s.elem_count_histogram, + 'range_bounds_histogram', s.range_bounds_histogram::text, + 'range_empty_frac', s.range_empty_frac, + 'range_length_histogram', s.range_length_histogram::text) AS r +WHERE s.schemaname = 'stats_import' +AND s.tablename IN ('test', 'is_odd') +ORDER BY s.tablename, s.attname, s.inherited; + +SELECT c.relname, COUNT(*) AS num_stats +FROM pg_class AS c +JOIN pg_statistic s ON s.starelid = c.oid +WHERE c.relnamespace = 'stats_import'::regnamespace +AND c.relname IN ('test', 'test_clone', 'is_odd', 'is_odd_clone') +GROUP BY c.relname +ORDER BY c.relname; + +-- check test minus test_clone +SELECT + a.attname, s.stainherit, s.stanullfrac, s.stawidth, s.stadistinct, + s.stakind1, s.stakind2, s.stakind3, s.stakind4, s.stakind5, + s.staop1, s.staop2, s.staop3, s.staop4, s.staop5, + s.stacoll1, s.stacoll2, s.stacoll3, s.stacoll4, s.stacoll5, + s.stanumbers1, s.stanumbers2, s.stanumbers3, s.stanumbers4, s.stanumbers5, + s.stavalues1::text AS sv1, s.stavalues2::text AS sv2, + s.stavalues3::text AS sv3, s.stavalues4::text AS sv4, + s.stavalues5::text AS sv5, 'test' AS direction +FROM pg_statistic s +JOIN pg_attribute a ON a.attrelid = s.starelid AND a.attnum = s.staattnum +WHERE s.starelid = 'stats_import.test'::regclass +EXCEPT +SELECT + a.attname, s.stainherit, s.stanullfrac, s.stawidth, s.stadistinct, + s.stakind1, s.stakind2, s.stakind3, s.stakind4, s.stakind5, + s.staop1, s.staop2, s.staop3, s.staop4, s.staop5, + s.stacoll1, s.stacoll2, s.stacoll3, s.stacoll4, s.stacoll5, + s.stanumbers1, s.stanumbers2, s.stanumbers3, s.stanumbers4, s.stanumbers5, + s.stavalues1::text AS sv1, s.stavalues2::text AS sv2, + s.stavalues3::text AS sv3, s.stavalues4::text AS sv4, + s.stavalues5::text AS sv5, 'test' AS direction +FROM pg_statistic s +JOIN pg_attribute a ON a.attrelid = s.starelid AND a.attnum = s.staattnum +WHERE s.starelid = 'stats_import.test_clone'::regclass; + +-- check test_clone minus test +SELECT + a.attname, s.stainherit, s.stanullfrac, s.stawidth, s.stadistinct, + s.stakind1, s.stakind2, s.stakind3, s.stakind4, s.stakind5, + s.staop1, s.staop2, s.staop3, s.staop4, s.staop5, + s.stacoll1, s.stacoll2, s.stacoll3, s.stacoll4, s.stacoll5, + s.stanumbers1, s.stanumbers2, s.stanumbers3, s.stanumbers4, s.stanumbers5, + s.stavalues1::text AS sv1, s.stavalues2::text AS sv2, + s.stavalues3::text AS sv3, s.stavalues4::text AS sv4, + s.stavalues5::text AS sv5, 'test_clone' AS direction +FROM pg_statistic s +JOIN pg_attribute a ON a.attrelid = s.starelid AND a.attnum = s.staattnum +WHERE s.starelid = 'stats_import.test_clone'::regclass +EXCEPT +SELECT + a.attname, s.stainherit, s.stanullfrac, s.stawidth, s.stadistinct, + s.stakind1, s.stakind2, s.stakind3, s.stakind4, s.stakind5, + s.staop1, s.staop2, s.staop3, s.staop4, s.staop5, + s.stacoll1, s.stacoll2, s.stacoll3, s.stacoll4, s.stacoll5, + s.stanumbers1, s.stanumbers2, s.stanumbers3, s.stanumbers4, s.stanumbers5, + s.stavalues1::text AS sv1, s.stavalues2::text AS sv2, + s.stavalues3::text AS sv3, s.stavalues4::text AS sv4, + s.stavalues5::text AS sv5, 'test_clone' AS direction +FROM pg_statistic s +JOIN pg_attribute a ON a.attrelid = s.starelid AND a.attnum = s.staattnum +WHERE s.starelid = 'stats_import.test'::regclass; + +-- check is_odd minus is_odd_clone +SELECT + a.attname, s.stainherit, s.stanullfrac, s.stawidth, s.stadistinct, + s.stakind1, s.stakind2, s.stakind3, s.stakind4, s.stakind5, + s.staop1, s.staop2, s.staop3, s.staop4, s.staop5, + s.stacoll1, s.stacoll2, s.stacoll3, s.stacoll4, s.stacoll5, + s.stanumbers1, s.stanumbers2, s.stanumbers3, s.stanumbers4, s.stanumbers5, + s.stavalues1::text AS sv1, s.stavalues2::text AS sv2, + s.stavalues3::text AS sv3, s.stavalues4::text AS sv4, + s.stavalues5::text AS sv5, 'is_odd' AS direction +FROM pg_statistic s +JOIN pg_attribute a ON a.attrelid = s.starelid AND a.attnum = s.staattnum +WHERE s.starelid = 'stats_import.is_odd'::regclass +EXCEPT +SELECT + a.attname, s.stainherit, s.stanullfrac, s.stawidth, s.stadistinct, + s.stakind1, s.stakind2, s.stakind3, s.stakind4, s.stakind5, + s.staop1, s.staop2, s.staop3, s.staop4, s.staop5, + s.stacoll1, s.stacoll2, s.stacoll3, s.stacoll4, s.stacoll5, + s.stanumbers1, s.stanumbers2, s.stanumbers3, s.stanumbers4, s.stanumbers5, + s.stavalues1::text AS sv1, s.stavalues2::text AS sv2, + s.stavalues3::text AS sv3, s.stavalues4::text AS sv4, + s.stavalues5::text AS sv5, 'is_odd' AS direction +FROM pg_statistic s +JOIN pg_attribute a ON a.attrelid = s.starelid AND a.attnum = s.staattnum +WHERE s.starelid = 'stats_import.is_odd_clone'::regclass; + +-- check is_odd_clone minus is_odd +SELECT + a.attname, s.stainherit, s.stanullfrac, s.stawidth, s.stadistinct, + s.stakind1, s.stakind2, s.stakind3, s.stakind4, s.stakind5, + s.staop1, s.staop2, s.staop3, s.staop4, s.staop5, + s.stacoll1, s.stacoll2, s.stacoll3, s.stacoll4, s.stacoll5, + s.stanumbers1, s.stanumbers2, s.stanumbers3, s.stanumbers4, s.stanumbers5, + s.stavalues1::text AS sv1, s.stavalues2::text AS sv2, + s.stavalues3::text AS sv3, s.stavalues4::text AS sv4, + s.stavalues5::text AS sv5, 'is_odd_clone' AS direction +FROM pg_statistic s +JOIN pg_attribute a ON a.attrelid = s.starelid AND a.attnum = s.staattnum +WHERE s.starelid = 'stats_import.is_odd_clone'::regclass +EXCEPT +SELECT + a.attname, s.stainherit, s.stanullfrac, s.stawidth, s.stadistinct, + s.stakind1, s.stakind2, s.stakind3, s.stakind4, s.stakind5, + s.staop1, s.staop2, s.staop3, s.staop4, s.staop5, + s.stacoll1, s.stacoll2, s.stacoll3, s.stacoll4, s.stacoll5, + s.stanumbers1, s.stanumbers2, s.stanumbers3, s.stanumbers4, s.stanumbers5, + s.stavalues1::text AS sv1, s.stavalues2::text AS sv2, + s.stavalues3::text AS sv3, s.stavalues4::text AS sv4, + s.stavalues5::text AS sv5, 'is_odd_clone' AS direction +FROM pg_statistic s +JOIN pg_attribute a ON a.attrelid = s.starelid AND a.attnum = s.staattnum +WHERE s.starelid = 'stats_import.is_odd'::regclass; + DROP SCHEMA stats_import CASCADE; From 14ec8bac8572746cbfe40bdb06ed089abbe89e74 Mon Sep 17 00:00:00 2001 From: Jeff Davis Date: Fri, 22 Nov 2024 12:07:46 -0800 Subject: [PATCH 10/54] Disallow modifying statistics on system columns. Reported-by: Heikki Linnakangas Discussion: https://postgr.es/m/df3e1c41-4e6c-40ad-9636-98deefe488cd@iki.fi --- src/backend/statistics/attribute_stats.c | 14 ++++++++++++++ src/test/regress/expected/stats_import.out | 15 +++++++++++++++ src/test/regress/sql/stats_import.sql | 15 +++++++++++++++ 3 files changed, 44 insertions(+) diff --git a/src/backend/statistics/attribute_stats.c b/src/backend/statistics/attribute_stats.c index 0da3bc2e3a7..4080a009a5e 100644 --- a/src/backend/statistics/attribute_stats.c +++ b/src/backend/statistics/attribute_stats.c @@ -161,6 +161,13 @@ attribute_statistics_update(FunctionCallInfo fcinfo, int elevel) stats_check_required_arg(fcinfo, attarginfo, ATTNAME_ARG); attname = PG_GETARG_NAME(ATTNAME_ARG); attnum = get_attnum(reloid, NameStr(*attname)); + + if (attnum < 0) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot modify statistics on system column \"%s\"", + NameStr(*attname)))); + if (attnum == InvalidAttrNumber) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_COLUMN), @@ -865,6 +872,13 @@ pg_clear_attribute_stats(PG_FUNCTION_ARGS) stats_check_required_arg(fcinfo, attarginfo, ATTNAME_ARG); attname = PG_GETARG_NAME(ATTNAME_ARG); attnum = get_attnum(reloid, NameStr(*attname)); + + if (attnum < 0) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot clear statistics on system column \"%s\"", + NameStr(*attname)))); + if (attnum == InvalidAttrNumber) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_COLUMN), diff --git a/src/test/regress/expected/stats_import.out b/src/test/regress/expected/stats_import.out index 9186fc01ecc..aab862c97c7 100644 --- a/src/test/regress/expected/stats_import.out +++ b/src/test/regress/expected/stats_import.out @@ -195,6 +195,15 @@ SELECT pg_catalog.pg_set_attribute_stats( avg_width => 2::integer, n_distinct => 0.3::real); ERROR: "relation" cannot be NULL +-- error: attribute is system column +SELECT pg_catalog.pg_set_attribute_stats( + relation => 'stats_import.test'::regclass, + attname => 'xmin'::name, + inherited => false::boolean, + null_frac => 0.1::real, + avg_width => 2::integer, + n_distinct => 0.3::real); +ERROR: cannot modify statistics on system column "xmin" -- error: attname doesn't exist SELECT pg_catalog.pg_set_attribute_stats( relation => 'stats_import.test'::regclass, @@ -204,6 +213,12 @@ SELECT pg_catalog.pg_set_attribute_stats( avg_width => 2::integer, n_distinct => 0.3::real); ERROR: column "nope" of relation "test" does not exist +-- error: attribute is system column +SELECT pg_catalog.pg_clear_attribute_stats( + relation => 'stats_import.test'::regclass, + attname => 'ctid'::name, + inherited => false::boolean); +ERROR: cannot clear statistics on system column "ctid" -- error: attname doesn't exist SELECT pg_catalog.pg_clear_attribute_stats( relation => 'stats_import.test'::regclass, diff --git a/src/test/regress/sql/stats_import.sql b/src/test/regress/sql/stats_import.sql index c7d5e017d90..31455b58c1d 100644 --- a/src/test/regress/sql/stats_import.sql +++ b/src/test/regress/sql/stats_import.sql @@ -145,6 +145,15 @@ SELECT pg_catalog.pg_set_attribute_stats( avg_width => 2::integer, n_distinct => 0.3::real); +-- error: attribute is system column +SELECT pg_catalog.pg_set_attribute_stats( + relation => 'stats_import.test'::regclass, + attname => 'xmin'::name, + inherited => false::boolean, + null_frac => 0.1::real, + avg_width => 2::integer, + n_distinct => 0.3::real); + -- error: attname doesn't exist SELECT pg_catalog.pg_set_attribute_stats( relation => 'stats_import.test'::regclass, @@ -154,6 +163,12 @@ SELECT pg_catalog.pg_set_attribute_stats( avg_width => 2::integer, n_distinct => 0.3::real); +-- error: attribute is system column +SELECT pg_catalog.pg_clear_attribute_stats( + relation => 'stats_import.test'::regclass, + attname => 'ctid'::name, + inherited => false::boolean); + -- error: attname doesn't exist SELECT pg_catalog.pg_clear_attribute_stats( relation => 'stats_import.test'::regclass, From 1a42606927d29d24ce6bbc48a23341c96fa8a628 Mon Sep 17 00:00:00 2001 From: Jeff Davis Date: Tue, 10 Dec 2024 16:30:37 -0800 Subject: [PATCH 11/54] Use in-place updates for pg_restore_relation_stats(). This matches the behavior of vac_update_relstats(), which is important to avoid bloating pg_class. Author: Corey Huinker Discussion: https://postgr.es/m/CADkLM=fc3je+ufv3gsHqjjSSf+t8674RXpuXW62EL55MUEQd-g@mail.gmail.com --- doc/src/sgml/func.sgml | 8 + src/backend/statistics/relation_stats.c | 188 ++++++++++++++------- src/test/regress/expected/stats_import.out | 61 +++++++ src/test/regress/sql/stats_import.sql | 37 ++++ 4 files changed, 229 insertions(+), 65 deletions(-) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 8f22eb40cf3..a831c720ef1 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -30054,6 +30054,14 @@ postgres=# SELECT '0/0'::pg_lsn + pd.segment_number * ps.setting::int + :offset function is to maintain a consistent function signature to avoid errors when restoring statistics from previous versions. + + To match the behavior of and when updating relation statistics, + pg_restore_relation_stats() does not follow MVCC + transactional semantics (see ). New relation + statistics may be durable even if the transaction aborts, and the + changes are not isolated from other transactions. + Arguments are passed as pairs of argname and argvalue, where diff --git a/src/backend/statistics/relation_stats.c b/src/backend/statistics/relation_stats.c index 5a2aabc921e..bd336a1f2b3 100644 --- a/src/backend/statistics/relation_stats.c +++ b/src/backend/statistics/relation_stats.c @@ -20,6 +20,7 @@ #include "access/heapam.h" #include "catalog/indexing.h" #include "statistics/stat_utils.h" +#include "utils/fmgroids.h" #include "utils/fmgrprotos.h" #include "utils/syscache.h" @@ -50,53 +51,28 @@ static struct StatsArgInfo relarginfo[] = [NUM_RELATION_STATS_ARGS] = {0} }; -static bool relation_statistics_update(FunctionCallInfo fcinfo, int elevel); +static bool relation_statistics_update(FunctionCallInfo fcinfo, int elevel, + bool inplace); /* * Internal function for modifying statistics for a relation. */ static bool -relation_statistics_update(FunctionCallInfo fcinfo, int elevel) +relation_statistics_update(FunctionCallInfo fcinfo, int elevel, bool inplace) { Oid reloid; Relation crel; - HeapTuple ctup; - Form_pg_class pgcform; - int replaces[3] = {0}; - Datum values[3] = {0}; - bool nulls[3] = {0}; - int ncols = 0; - TupleDesc tupdesc; + int32 relpages = DEFAULT_RELPAGES; + bool update_relpages = false; + float reltuples = DEFAULT_RELTUPLES; + bool update_reltuples = false; + int32 relallvisible = DEFAULT_RELALLVISIBLE; + bool update_relallvisible = false; bool result = true; - stats_check_required_arg(fcinfo, relarginfo, RELATION_ARG); - reloid = PG_GETARG_OID(RELATION_ARG); - - stats_lock_check_privileges(reloid); - - /* - * Take RowExclusiveLock on pg_class, consistent with - * vac_update_relstats(). - */ - crel = table_open(RelationRelationId, RowExclusiveLock); - - tupdesc = RelationGetDescr(crel); - ctup = SearchSysCacheCopy1(RELOID, ObjectIdGetDatum(reloid)); - if (!HeapTupleIsValid(ctup)) - { - ereport(elevel, - (errcode(ERRCODE_OBJECT_IN_USE), - errmsg("pg_class entry for relid %u not found", reloid))); - table_close(crel, RowExclusiveLock); - return false; - } - - pgcform = (Form_pg_class) GETSTRUCT(ctup); - - /* relpages */ if (!PG_ARGISNULL(RELPAGES_ARG)) { - int32 relpages = PG_GETARG_INT32(RELPAGES_ARG); + relpages = PG_GETARG_INT32(RELPAGES_ARG); /* * Partitioned tables may have relpages=-1. Note: for relations with @@ -110,17 +86,13 @@ relation_statistics_update(FunctionCallInfo fcinfo, int elevel) errmsg("relpages cannot be < -1"))); result = false; } - else if (relpages != pgcform->relpages) - { - replaces[ncols] = Anum_pg_class_relpages; - values[ncols] = Int32GetDatum(relpages); - ncols++; - } + else + update_relpages = true; } if (!PG_ARGISNULL(RELTUPLES_ARG)) { - float reltuples = PG_GETARG_FLOAT4(RELTUPLES_ARG); + reltuples = PG_GETARG_FLOAT4(RELTUPLES_ARG); if (reltuples < -1.0) { @@ -129,18 +101,13 @@ relation_statistics_update(FunctionCallInfo fcinfo, int elevel) errmsg("reltuples cannot be < -1.0"))); result = false; } - else if (reltuples != pgcform->reltuples) - { - replaces[ncols] = Anum_pg_class_reltuples; - values[ncols] = Float4GetDatum(reltuples); - ncols++; - } - + else + update_reltuples = true; } if (!PG_ARGISNULL(RELALLVISIBLE_ARG)) { - int32 relallvisible = PG_GETARG_INT32(RELALLVISIBLE_ARG); + relallvisible = PG_GETARG_INT32(RELALLVISIBLE_ARG); if (relallvisible < 0) { @@ -149,23 +116,114 @@ relation_statistics_update(FunctionCallInfo fcinfo, int elevel) errmsg("relallvisible cannot be < 0"))); result = false; } - else if (relallvisible != pgcform->relallvisible) + else + update_relallvisible = true; + } + + stats_check_required_arg(fcinfo, relarginfo, RELATION_ARG); + reloid = PG_GETARG_OID(RELATION_ARG); + + stats_lock_check_privileges(reloid); + + /* + * Take RowExclusiveLock on pg_class, consistent with + * vac_update_relstats(). + */ + crel = table_open(RelationRelationId, RowExclusiveLock); + + if (inplace) + { + HeapTuple ctup = NULL; + ScanKeyData key[1]; + Form_pg_class pgcform; + void *inplace_state = NULL; + bool dirty = false; + + ScanKeyInit(&key[0], Anum_pg_class_oid, BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(reloid)); + systable_inplace_update_begin(crel, ClassOidIndexId, true, NULL, 1, key, + &ctup, &inplace_state); + if (!HeapTupleIsValid(ctup)) + elog(ERROR, "pg_class entry for relid %u vanished while updating statistics", + reloid); + pgcform = (Form_pg_class) GETSTRUCT(ctup); + + if (update_relpages && pgcform->relpages != relpages) { - replaces[ncols] = Anum_pg_class_relallvisible; - values[ncols] = Int32GetDatum(relallvisible); - ncols++; + pgcform->relpages = relpages; + dirty = true; } - } + if (update_reltuples && pgcform->reltuples != reltuples) + { + pgcform->reltuples = reltuples; + dirty = true; + } + if (update_relallvisible && pgcform->relallvisible != relallvisible) + { + pgcform->relallvisible = relallvisible; + dirty = true; + } + + if (dirty) + systable_inplace_update_finish(inplace_state, ctup); + else + systable_inplace_update_cancel(inplace_state); - /* only update pg_class if there is a meaningful change */ - if (ncols > 0) + heap_freetuple(ctup); + } + else { - HeapTuple newtup; + TupleDesc tupdesc = RelationGetDescr(crel); + HeapTuple ctup; + Form_pg_class pgcform; + int replaces[3] = {0}; + Datum values[3] = {0}; + bool nulls[3] = {0}; + int nreplaces = 0; + + ctup = SearchSysCache1(RELOID, ObjectIdGetDatum(reloid)); + if (!HeapTupleIsValid(ctup)) + { + ereport(elevel, + (errcode(ERRCODE_OBJECT_IN_USE), + errmsg("pg_class entry for relid %u not found", reloid))); + table_close(crel, RowExclusiveLock); + return false; + } + pgcform = (Form_pg_class) GETSTRUCT(ctup); + + if (update_relpages && relpages != pgcform->relpages) + { + replaces[nreplaces] = Anum_pg_class_relpages; + values[nreplaces] = Int32GetDatum(relpages); + nreplaces++; + } + + if (update_reltuples && reltuples != pgcform->reltuples) + { + replaces[nreplaces] = Anum_pg_class_reltuples; + values[nreplaces] = Float4GetDatum(reltuples); + nreplaces++; + } + + if (update_relallvisible && relallvisible != pgcform->relallvisible) + { + replaces[nreplaces] = Anum_pg_class_relallvisible; + values[nreplaces] = Int32GetDatum(relallvisible); + nreplaces++; + } + + if (nreplaces > 0) + { + HeapTuple newtup; + + newtup = heap_modify_tuple_by_cols(ctup, tupdesc, nreplaces, + replaces, values, nulls); + CatalogTupleUpdate(crel, &newtup->t_self, newtup); + heap_freetuple(newtup); + } - newtup = heap_modify_tuple_by_cols(ctup, tupdesc, ncols, replaces, values, - nulls); - CatalogTupleUpdate(crel, &newtup->t_self, newtup); - heap_freetuple(newtup); + ReleaseSysCache(ctup); } /* release the lock, consistent with vac_update_relstats() */ @@ -180,7 +238,7 @@ relation_statistics_update(FunctionCallInfo fcinfo, int elevel) Datum pg_set_relation_stats(PG_FUNCTION_ARGS) { - relation_statistics_update(fcinfo, ERROR); + relation_statistics_update(fcinfo, ERROR, false); PG_RETURN_VOID(); } @@ -204,7 +262,7 @@ pg_clear_relation_stats(PG_FUNCTION_ARGS) newfcinfo->args[3].value = DEFAULT_RELALLVISIBLE; newfcinfo->args[3].isnull = false; - relation_statistics_update(newfcinfo, ERROR); + relation_statistics_update(newfcinfo, ERROR, false); PG_RETURN_VOID(); } @@ -222,7 +280,7 @@ pg_restore_relation_stats(PG_FUNCTION_ARGS) relarginfo, WARNING)) result = false; - if (!relation_statistics_update(positional_fcinfo, WARNING)) + if (!relation_statistics_update(positional_fcinfo, WARNING, true)) result = false; PG_RETURN_BOOL(result); diff --git a/src/test/regress/expected/stats_import.out b/src/test/regress/expected/stats_import.out index aab862c97c7..fb50da1cd83 100644 --- a/src/test/regress/expected/stats_import.out +++ b/src/test/regress/expected/stats_import.out @@ -105,6 +105,47 @@ WHERE oid = 'stats_import.test'::regclass; 18 | 401 | 5 (1 row) +-- test MVCC behavior: changes do not persist after abort (in contrast +-- to pg_restore_relation_stats(), which uses in-place updates). +BEGIN; +SELECT + pg_catalog.pg_set_relation_stats( + relation => 'stats_import.test'::regclass, + relpages => NULL::integer, + reltuples => 4000.0::real, + relallvisible => 4::integer); + pg_set_relation_stats +----------------------- + +(1 row) + +ABORT; +SELECT relpages, reltuples, relallvisible +FROM pg_class +WHERE oid = 'stats_import.test'::regclass; + relpages | reltuples | relallvisible +----------+-----------+--------------- + 18 | 401 | 5 +(1 row) + +BEGIN; +SELECT + pg_catalog.pg_clear_relation_stats( + 'stats_import.test'::regclass); + pg_clear_relation_stats +------------------------- + +(1 row) + +ABORT; +SELECT relpages, reltuples, relallvisible +FROM pg_class +WHERE oid = 'stats_import.test'::regclass; + relpages | reltuples | relallvisible +----------+-----------+--------------- + 18 | 401 | 5 +(1 row) + -- clear SELECT pg_catalog.pg_clear_relation_stats( @@ -705,6 +746,25 @@ WHERE oid = 'stats_import.test'::regclass; (1 row) -- ok: just relpages +SELECT pg_restore_relation_stats( + 'relation', 'stats_import.test'::regclass, + 'version', 150000::integer, + 'relpages', '15'::integer); + pg_restore_relation_stats +--------------------------- + t +(1 row) + +SELECT relpages, reltuples, relallvisible +FROM pg_class +WHERE oid = 'stats_import.test'::regclass; + relpages | reltuples | relallvisible +----------+-----------+--------------- + 15 | 400 | 4 +(1 row) + +-- test non-MVCC behavior: new value should persist after abort +BEGIN; SELECT pg_restore_relation_stats( 'relation', 'stats_import.test'::regclass, 'version', 150000::integer, @@ -714,6 +774,7 @@ SELECT pg_restore_relation_stats( t (1 row) +ABORT; SELECT relpages, reltuples, relallvisible FROM pg_class WHERE oid = 'stats_import.test'::regclass; diff --git a/src/test/regress/sql/stats_import.sql b/src/test/regress/sql/stats_import.sql index 31455b58c1d..d3058bf8f6b 100644 --- a/src/test/regress/sql/stats_import.sql +++ b/src/test/regress/sql/stats_import.sql @@ -76,6 +76,31 @@ SELECT relpages, reltuples, relallvisible FROM pg_class WHERE oid = 'stats_import.test'::regclass; +-- test MVCC behavior: changes do not persist after abort (in contrast +-- to pg_restore_relation_stats(), which uses in-place updates). +BEGIN; +SELECT + pg_catalog.pg_set_relation_stats( + relation => 'stats_import.test'::regclass, + relpages => NULL::integer, + reltuples => 4000.0::real, + relallvisible => 4::integer); +ABORT; + +SELECT relpages, reltuples, relallvisible +FROM pg_class +WHERE oid = 'stats_import.test'::regclass; + +BEGIN; +SELECT + pg_catalog.pg_clear_relation_stats( + 'stats_import.test'::regclass); +ABORT; + +SELECT relpages, reltuples, relallvisible +FROM pg_class +WHERE oid = 'stats_import.test'::regclass; + -- clear SELECT pg_catalog.pg_clear_relation_stats( @@ -565,10 +590,22 @@ FROM pg_class WHERE oid = 'stats_import.test'::regclass; -- ok: just relpages +SELECT pg_restore_relation_stats( + 'relation', 'stats_import.test'::regclass, + 'version', 150000::integer, + 'relpages', '15'::integer); + +SELECT relpages, reltuples, relallvisible +FROM pg_class +WHERE oid = 'stats_import.test'::regclass; + +-- test non-MVCC behavior: new value should persist after abort +BEGIN; SELECT pg_restore_relation_stats( 'relation', 'stats_import.test'::regclass, 'version', 150000::integer, 'relpages', '16'::integer); +ABORT; SELECT relpages, reltuples, relallvisible FROM pg_class From 1091da0564477cf7a7f85f16df0bb88da10b76d9 Mon Sep 17 00:00:00 2001 From: Jeff Davis Date: Thu, 20 Feb 2025 01:29:06 -0800 Subject: [PATCH 12/54] Transfer statistics during pg_upgrade. Add support to pg_dump for dumping stats, and use that during pg_upgrade so that statistics are transferred during upgrade. In most cases this removes the need for a costly re-analyze after upgrade. Some statistics are not transferred, such as extended statistics or statistics with a custom stakind. Now pg_dump accepts the options --schema-only, --no-schema, --data-only, --no-data, --statistics-only, and --no-statistics; which allow all combinations of schema, data, and/or stats. The options are named this way to preserve compatibility with the previous --schema-only and --data-only options. Statistics are in SECTION_DATA, unless the object itself is in SECTION_POST_DATA. The stats are represented as calls to pg_restore_relation_stats() and pg_restore_attribute_stats(). Author: Corey Huinker, Jeff Davis Reviewed-by: Jian He Discussion: https://postgr.es/m/CADkLM=fzX7QX6r78fShWDjNN3Vcr4PVAnvXxQ4DiGy6V=0bCUA@mail.gmail.com Discussion: https://postgr.es/m/CADkLM%3DcB0rF3p_FuWRTMSV0983ihTRpsH%2BOCpNyiqE7Wk0vUWA%40mail.gmail.com --- doc/src/sgml/ref/pg_dump.sgml | 75 +++- doc/src/sgml/ref/pg_dumpall.sgml | 41 +- doc/src/sgml/ref/pg_restore.sgml | 48 +- doc/src/sgml/ref/pgupgrade.sgml | 9 + src/bin/pg_dump/pg_backup.h | 2 + src/bin/pg_dump/pg_backup_archiver.c | 83 ++-- src/bin/pg_dump/pg_backup_archiver.h | 3 +- src/bin/pg_dump/pg_backup_directory.c | 2 +- src/bin/pg_dump/pg_dump.c | 412 +++++++++++++++++- src/bin/pg_dump/pg_dump.h | 12 + src/bin/pg_dump/pg_dump_sort.c | 37 +- src/bin/pg_dump/pg_dumpall.c | 23 +- src/bin/pg_dump/pg_restore.c | 24 +- src/bin/pg_dump/t/001_basic.pl | 18 + src/bin/pg_dump/t/002_pg_dump.pl | 106 ++++- src/bin/pg_upgrade/dump.c | 3 +- src/bin/pg_upgrade/option.c | 7 + src/bin/pg_upgrade/pg_upgrade.h | 1 + src/bin/pg_upgrade/t/002_pg_upgrade.pl | 7 + .../perl/PostgreSQL/Test/AdjustUpgrade.pm | 9 + src/test/recovery/t/027_stream_regress.pl | 4 +- src/tools/pgindent/typedefs.list | 1 + 22 files changed, 850 insertions(+), 77 deletions(-) diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml index 5ba167b33e7..7ccc41ab1f5 100644 --- a/doc/src/sgml/ref/pg_dump.sgml +++ b/doc/src/sgml/ref/pg_dump.sgml @@ -131,7 +131,7 @@ PostgreSQL documentation - Dump only the data, not the schema (data definitions). + Dump only the data, not the schema (data definitions) or statistics. Table data, large objects, and sequence values are dumped. @@ -149,13 +149,15 @@ PostgreSQL documentation Include large objects in the dump. This is the default behavior - except when , , or - is specified. The - switch is therefore only useful to add large objects to dumps - where a specific schema or table has been requested. Note that - large objects are considered data and therefore will be included when - is used, but not - when is. + except when , , + , , or + is specified. The + switch is therefore only useful to add large objects to dumps where a + specific schema or table has been requested. Note that large objects + are considered data and therefore will be included when + is used, but not when + or + is. @@ -524,10 +526,11 @@ PostgreSQL documentation - Dump only the object definitions (schema), not data. + Dump only the object definitions (schema), not data or statistics. - This option is the inverse of . + This option is mutually exclusive to + and . It is similar to, but for historical reasons not identical to, specifying . @@ -660,6 +663,17 @@ PostgreSQL documentation + + + + + Dump only the statistics, not the schema (data definitions) or data. + Statistics for tables, materialized views, and indexes are dumped. + + + + + [:detail] @@ -749,7 +763,8 @@ PostgreSQL documentation - This option is relevant only when creating a data-only dump. + This option is relevant only when creating a dump that includes data + but does not include schema. It instructs pg_dump to include commands to temporarily disable triggers on the target tables while the data is restored. Use this if you have referential @@ -841,7 +856,8 @@ PostgreSQL documentation though you do not need the data in it. - To exclude data for all tables in the database, see . + To exclude data for all tables in the database, see + or . @@ -1088,6 +1104,15 @@ PostgreSQL documentation + + + + + Do not dump data. + + + + @@ -1106,6 +1131,24 @@ PostgreSQL documentation + + + + + Do not dump schema (data definitions). + + + + + + + + + Do not dump statistics. + + + + @@ -1267,9 +1310,11 @@ PostgreSQL documentation The data section contains actual table data, large-object - contents, and sequence values. + contents, statitistics for tables and materialized views and + sequence values. Post-data items include definitions of indexes, triggers, rules, - and constraints other than validated check constraints. + statistics for indexes, and constraints other than validated check + constraints. Pre-data items include all other data definition items. @@ -1612,7 +1657,7 @@ CREATE DATABASE foo WITH TEMPLATE template0; - When a data-only dump is chosen and the option + When a dump without schema is chosen and the option is used, pg_dump emits commands to disable triggers on user tables before inserting the data, and then commands to re-enable them after the data has been diff --git a/doc/src/sgml/ref/pg_dumpall.sgml b/doc/src/sgml/ref/pg_dumpall.sgml index 84e38ef2115..99e8e0316de 100644 --- a/doc/src/sgml/ref/pg_dumpall.sgml +++ b/doc/src/sgml/ref/pg_dumpall.sgml @@ -91,7 +91,7 @@ PostgreSQL documentation - Dump only the data, not the schema (data definitions). + Dump only the data, not the schema (data definitions) or statistics. @@ -275,6 +275,16 @@ exclude database PATTERN + + + + + Dump only the statistics, not the schema (data definitions) or data. + Statistics for tables, materialized views, and indexes are dumped. + + + + @@ -317,7 +327,7 @@ exclude database PATTERN - This option is relevant only when creating a data-only dump. + This option is relevant only when creating a dump with data and without schema. It instructs pg_dumpall to include commands to temporarily disable triggers on the target tables while the data is restored. Use this if you have referential @@ -432,6 +442,15 @@ exclude database PATTERN + + + + + Do not dump data. + + + + @@ -457,6 +476,15 @@ exclude database PATTERN + + + + + Do not dump schema (data definitions). + + + + @@ -466,6 +494,15 @@ exclude database PATTERN + + + + + Do not dump statistics. + + + + diff --git a/doc/src/sgml/ref/pg_restore.sgml b/doc/src/sgml/ref/pg_restore.sgml index 8e221d89d28..6c6e5767fbd 100644 --- a/doc/src/sgml/ref/pg_restore.sgml +++ b/doc/src/sgml/ref/pg_restore.sgml @@ -106,7 +106,7 @@ PostgreSQL documentation - Restore only the data, not the schema (data definitions). + Restore only the data, not the schema (data definitions) or statistics. Table data, large objects, and sequence values are restored, if present in the archive. @@ -495,10 +495,11 @@ PostgreSQL documentation to the extent that schema entries are present in the archive. - This option is the inverse of . + This option is mutually exclusive of + and . It is similar to, but for historical reasons not identical to, specifying - . + . (Do not confuse this with the option, which @@ -611,6 +612,15 @@ PostgreSQL documentation + + + + + Restore only the statistics, not schema (data definitions) or data. + + + + @@ -629,7 +639,7 @@ PostgreSQL documentation - This option is relevant only when performing a data-only restore. + This option is relevant only when performing a restore without schema. It instructs pg_restore to execute commands to temporarily disable triggers on the target tables while the data is restored. Use this if you have referential @@ -693,6 +703,16 @@ PostgreSQL documentation + + + + + Do not output commands to restore data, even if the archive + contains them. + + + + @@ -725,6 +745,16 @@ PostgreSQL documentation + + + + + Do not output commands to restore schema (data definitions), even if + the archive contains them. + + + + @@ -735,6 +765,16 @@ PostgreSQL documentation + + + + + Do not output commands to restore statistics, even if the archive + contains them. + + + + diff --git a/doc/src/sgml/ref/pgupgrade.sgml b/doc/src/sgml/ref/pgupgrade.sgml index cdc37e91abe..6e0e94a7a67 100644 --- a/doc/src/sgml/ref/pgupgrade.sgml +++ b/doc/src/sgml/ref/pgupgrade.sgml @@ -153,6 +153,15 @@ PostgreSQL documentation + + + + + Do not restore statistics from the old cluster into the new cluster. + + + + options options diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h index 113b4bf8b62..329a8258214 100644 --- a/src/bin/pg_dump/pg_backup.h +++ b/src/bin/pg_dump/pg_backup.h @@ -160,6 +160,7 @@ typedef struct _restoreOptions /* flags derived from the user-settable flags */ bool dumpSchema; bool dumpData; + bool dumpStatistics; } RestoreOptions; typedef struct _dumpOptions @@ -209,6 +210,7 @@ typedef struct _dumpOptions /* flags derived from the user-settable flags */ bool dumpSchema; bool dumpData; + bool dumpStatistics; } DumpOptions; /* diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c index 8485ff72b11..a52acb248d5 100644 --- a/src/bin/pg_dump/pg_backup_archiver.c +++ b/src/bin/pg_dump/pg_backup_archiver.c @@ -46,6 +46,9 @@ #define TEXT_DUMP_HEADER "--\n-- PostgreSQL database dump\n--\n\n" #define TEXT_DUMPALL_HEADER "--\n-- PostgreSQL database cluster dump\n--\n\n" +#define TOC_PREFIX_NONE "" +#define TOC_PREFIX_DATA "Data for " +#define TOC_PREFIX_STATS "Statistics for " static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt, const pg_compress_specification compression_spec, @@ -53,7 +56,7 @@ static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt, SetupWorkerPtrType setupWorkerPtr, DataDirSyncMethod sync_method); static void _getObjectDescription(PQExpBuffer buf, const TocEntry *te); -static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData); +static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, const char *pfx); static void _doSetFixedOutputState(ArchiveHandle *AH); static void _doSetSessionAuth(ArchiveHandle *AH, const char *user); static void _reconnectToDB(ArchiveHandle *AH, const char *dbname); @@ -148,6 +151,7 @@ InitDumpOptions(DumpOptions *opts) opts->dumpSections = DUMP_UNSECTIONED; opts->dumpSchema = true; opts->dumpData = true; + opts->dumpStatistics = true; } /* @@ -168,9 +172,10 @@ dumpOptionsFromRestoreOptions(RestoreOptions *ropt) dopt->outputClean = ropt->dropSchema; dopt->dumpData = ropt->dumpData; dopt->dumpSchema = ropt->dumpSchema; + dopt->dumpSections = ropt->dumpSections; + dopt->dumpStatistics = ropt->dumpStatistics; dopt->if_exists = ropt->if_exists; dopt->column_inserts = ropt->column_inserts; - dopt->dumpSections = ropt->dumpSections; dopt->aclsSkip = ropt->aclsSkip; dopt->outputSuperuser = ropt->superuser; dopt->outputCreateDB = ropt->createDB; @@ -418,8 +423,8 @@ RestoreArchive(Archive *AHX) } /* - * Work out if we have an implied data-only restore. This can happen if - * the dump was data only or if the user has used a toc list to exclude + * Work out if we have an implied schema-less restore. This can happen if + * the dump excluded the schema or the user has used a toc list to exclude * all of the schema data. All we do is look for schema entries - if none * are found then we unset the dumpSchema flag. * @@ -428,20 +433,20 @@ RestoreArchive(Archive *AHX) */ if (ropt->dumpSchema) { - int impliedDataOnly = 1; + bool no_schema_found = true; for (te = AH->toc->next; te != AH->toc; te = te->next) { if ((te->reqs & REQ_SCHEMA) != 0) - { /* It's schema, and it's wanted */ - impliedDataOnly = 0; + { + no_schema_found = false; break; } } - if (impliedDataOnly) + if (no_schema_found) { ropt->dumpSchema = false; - pg_log_info("implied data-only restore"); + pg_log_info("implied no-schema restore"); } } @@ -750,7 +755,7 @@ RestoreArchive(Archive *AHX) for (te = AH->toc->next; te != AH->toc; te = te->next) { - if ((te->reqs & (REQ_SCHEMA | REQ_DATA)) == 0) + if ((te->reqs & (REQ_SCHEMA | REQ_DATA | REQ_STATS)) == 0) continue; /* ignore if not to be dumped at all */ switch (_tocEntryRestorePass(te)) @@ -771,7 +776,7 @@ RestoreArchive(Archive *AHX) { for (te = AH->toc->next; te != AH->toc; te = te->next) { - if ((te->reqs & (REQ_SCHEMA | REQ_DATA)) != 0 && + if ((te->reqs & (REQ_SCHEMA | REQ_DATA | REQ_STATS)) != 0 && _tocEntryRestorePass(te) == RESTORE_PASS_ACL) (void) restore_toc_entry(AH, te, false); } @@ -781,7 +786,7 @@ RestoreArchive(Archive *AHX) { for (te = AH->toc->next; te != AH->toc; te = te->next) { - if ((te->reqs & (REQ_SCHEMA | REQ_DATA)) != 0 && + if ((te->reqs & (REQ_SCHEMA | REQ_DATA | REQ_STATS)) != 0 && _tocEntryRestorePass(te) == RESTORE_PASS_POST_ACL) (void) restore_toc_entry(AH, te, false); } @@ -888,7 +893,7 @@ restore_toc_entry(ArchiveHandle *AH, TocEntry *te, bool is_parallel) pg_log_info("creating %s \"%s\"", te->desc, te->tag); - _printTocEntry(AH, te, false); + _printTocEntry(AH, te, TOC_PREFIX_NONE); defnDumped = true; if (strcmp(te->desc, "TABLE") == 0) @@ -957,7 +962,7 @@ restore_toc_entry(ArchiveHandle *AH, TocEntry *te, bool is_parallel) */ if (AH->PrintTocDataPtr != NULL) { - _printTocEntry(AH, te, true); + _printTocEntry(AH, te, TOC_PREFIX_DATA); if (strcmp(te->desc, "BLOBS") == 0 || strcmp(te->desc, "BLOB COMMENTS") == 0) @@ -1055,15 +1060,21 @@ restore_toc_entry(ArchiveHandle *AH, TocEntry *te, bool is_parallel) { /* If we haven't already dumped the defn part, do so now */ pg_log_info("executing %s %s", te->desc, te->tag); - _printTocEntry(AH, te, false); + _printTocEntry(AH, te, TOC_PREFIX_NONE); } } + /* + * If it has a statistics component that we want, then process that + */ + if ((reqs & REQ_STATS) != 0) + _printTocEntry(AH, te, TOC_PREFIX_STATS); + /* * If we emitted anything for this TOC entry, that counts as one action * against the transaction-size limit. Commit if it's time to. */ - if ((reqs & (REQ_SCHEMA | REQ_DATA)) != 0 && ropt->txn_size > 0) + if ((reqs & (REQ_SCHEMA | REQ_DATA | REQ_STATS)) != 0 && ropt->txn_size > 0) { if (++AH->txnCount >= ropt->txn_size) { @@ -1103,6 +1114,7 @@ NewRestoreOptions(void) opts->compression_spec.level = 0; opts->dumpSchema = true; opts->dumpData = true; + opts->dumpStatistics = true; return opts; } @@ -1348,7 +1360,7 @@ PrintTOCSummary(Archive *AHX) te->reqs = _tocEntryRequired(te, curSection, AH); /* Now, should we print it? */ if (ropt->verbose || - (te->reqs & (REQ_SCHEMA | REQ_DATA)) != 0) + (te->reqs & (REQ_SCHEMA | REQ_DATA | REQ_STATS)) != 0) { char *sanitized_name; char *sanitized_schema; @@ -2601,7 +2613,7 @@ WriteToc(ArchiveHandle *AH) tocCount = 0; for (te = AH->toc->next; te != AH->toc; te = te->next) { - if ((te->reqs & (REQ_SCHEMA | REQ_DATA | REQ_SPECIAL)) != 0) + if ((te->reqs & (REQ_SCHEMA | REQ_DATA | REQ_STATS | REQ_SPECIAL)) != 0) tocCount++; } @@ -2611,7 +2623,7 @@ WriteToc(ArchiveHandle *AH) for (te = AH->toc->next; te != AH->toc; te = te->next) { - if ((te->reqs & (REQ_SCHEMA | REQ_DATA | REQ_SPECIAL)) == 0) + if ((te->reqs & (REQ_SCHEMA | REQ_DATA | REQ_STATS | REQ_SPECIAL)) == 0) continue; WriteInt(AH, te->dumpId); @@ -2923,8 +2935,9 @@ StrictNamesCheck(RestoreOptions *ropt) * Determine whether we want to restore this TOC entry. * * Returns 0 if entry should be skipped, or some combination of the - * REQ_SCHEMA and REQ_DATA bits if we want to restore schema and/or data - * portions of this TOC entry, or REQ_SPECIAL if it's a special entry. + * REQ_SCHEMA, REQ_DATA, and REQ_STATS bits if we want to restore schema, data + * and/or statistics portions of this TOC entry, or REQ_SPECIAL if it's a + * special entry. */ static int _tocEntryRequired(TocEntry *te, teSection curSection, ArchiveHandle *AH) @@ -2938,6 +2951,14 @@ _tocEntryRequired(TocEntry *te, teSection curSection, ArchiveHandle *AH) strcmp(te->desc, "SEARCHPATH") == 0) return REQ_SPECIAL; + if (strcmp(te->desc, "STATISTICS DATA") == 0) + { + if (!ropt->dumpStatistics) + return 0; + else + res = REQ_STATS; + } + /* * DATABASE and DATABASE PROPERTIES also have a special rule: they are * restored in createDB mode, and not restored otherwise, independently of @@ -3011,6 +3032,10 @@ _tocEntryRequired(TocEntry *te, teSection curSection, ArchiveHandle *AH) if (ropt->no_subscriptions && strcmp(te->desc, "SUBSCRIPTION") == 0) return 0; + /* If it's statistics and we don't want statistics, maybe ignore it */ + if (!ropt->dumpStatistics && strcmp(te->desc, "STATISTICS DATA") == 0) + return 0; + /* Ignore it if section is not to be dumped/restored */ switch (curSection) { @@ -3040,6 +3065,7 @@ _tocEntryRequired(TocEntry *te, teSection curSection, ArchiveHandle *AH) */ if (strcmp(te->desc, "ACL") == 0 || strcmp(te->desc, "COMMENT") == 0 || + strcmp(te->desc, "STATISTICS DATA") == 0 || strcmp(te->desc, "SECURITY LABEL") == 0) { /* Database properties react to createDB, not selectivity options. */ @@ -3156,6 +3182,7 @@ _tocEntryRequired(TocEntry *te, teSection curSection, ArchiveHandle *AH) } } + /* * Determine whether the TOC entry contains schema and/or data components, * and mask off inapplicable REQ bits. If it had a dataDumper, assume @@ -3221,12 +3248,12 @@ _tocEntryRequired(TocEntry *te, teSection curSection, ArchiveHandle *AH) strncmp(te->tag, "LARGE OBJECT", 12) == 0) || (strcmp(te->desc, "SECURITY LABEL") == 0 && strncmp(te->tag, "LARGE OBJECT", 12) == 0)))) - res = res & REQ_SCHEMA; + res = res & (REQ_SCHEMA | REQ_STATS); } /* Mask it if we don't want schema */ if (!ropt->dumpSchema) - res = res & REQ_DATA; + res = res & (REQ_DATA | REQ_STATS); return res; } @@ -3790,7 +3817,7 @@ _getObjectDescription(PQExpBuffer buf, const TocEntry *te) * will remain at default, until the matching ACL TOC entry is restored. */ static void -_printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData) +_printTocEntry(ArchiveHandle *AH, TocEntry *te, const char *pfx) { RestoreOptions *ropt = AH->public.ropt; @@ -3809,16 +3836,10 @@ _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData) /* Emit header comment for item */ if (!AH->noTocComments) { - const char *pfx; char *sanitized_name; char *sanitized_schema; char *sanitized_owner; - if (isData) - pfx = "Data for "; - else - pfx = ""; - ahprintf(AH, "--\n"); if (AH->public.verbose) { @@ -4349,7 +4370,7 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate, if (next_work_item != NULL) { /* If not to be restored, don't waste time launching a worker */ - if ((next_work_item->reqs & (REQ_SCHEMA | REQ_DATA)) == 0) + if ((next_work_item->reqs & (REQ_SCHEMA | REQ_DATA | REQ_STATS)) == 0) { pg_log_info("skipping item %d %s %s", next_work_item->dumpId, diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h index ce5ed1dd395..a2064f471ed 100644 --- a/src/bin/pg_dump/pg_backup_archiver.h +++ b/src/bin/pg_dump/pg_backup_archiver.h @@ -209,7 +209,8 @@ typedef enum #define REQ_SCHEMA 0x01 /* want schema */ #define REQ_DATA 0x02 /* want data */ -#define REQ_SPECIAL 0x04 /* for special TOC entries */ +#define REQ_STATS 0x04 +#define REQ_SPECIAL 0x08 /* for special TOC entries */ struct _archiveHandle { diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c index bbaac2b4dc7..30c3b742735 100644 --- a/src/bin/pg_dump/pg_backup_directory.c +++ b/src/bin/pg_dump/pg_backup_directory.c @@ -755,7 +755,7 @@ _PrepParallelRestore(ArchiveHandle *AH) continue; /* We may ignore items not due to be restored */ - if ((te->reqs & REQ_DATA) == 0) + if ((te->reqs & (REQ_DATA | REQ_STATS)) == 0) continue; /* diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index a0c733c477d..895aeb877fc 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -375,6 +375,10 @@ main(int argc, char **argv) DataDirSyncMethod sync_method = DATA_DIR_SYNC_METHOD_FSYNC; bool data_only = false; bool schema_only = false; + bool statistics_only = false; + bool no_data = false; + bool no_schema = false; + bool no_statistics = false; static DumpOptions dopt; @@ -434,11 +438,15 @@ main(int argc, char **argv) {"section", required_argument, NULL, 5}, {"serializable-deferrable", no_argument, &dopt.serializable_deferrable, 1}, {"snapshot", required_argument, NULL, 6}, + {"statistics-only", no_argument, NULL, 18}, {"strict-names", no_argument, &strict_names, 1}, {"use-set-session-authorization", no_argument, &dopt.use_setsessauth, 1}, {"no-comments", no_argument, &dopt.no_comments, 1}, + {"no-data", no_argument, NULL, 19}, {"no-publications", no_argument, &dopt.no_publications, 1}, + {"no-schema", no_argument, NULL, 20}, {"no-security-labels", no_argument, &dopt.no_security_labels, 1}, + {"no-statistics", no_argument, NULL, 21}, {"no-subscriptions", no_argument, &dopt.no_subscriptions, 1}, {"no-toast-compression", no_argument, &dopt.no_toast_compression, 1}, {"no-unlogged-table-data", no_argument, &dopt.no_unlogged_table_data, 1}, @@ -485,7 +493,7 @@ main(int argc, char **argv) InitDumpOptions(&dopt); - while ((c = getopt_long(argc, argv, "abBcCd:e:E:f:F:h:j:n:N:Op:RsS:t:T:U:vwWxZ:", + while ((c = getopt_long(argc, argv, "abBcCd:e:E:f:F:h:j:n:N:Op:RsS:t:T:U:vwWxXZ:", long_options, &optindex)) != -1) { switch (c) @@ -692,11 +700,21 @@ main(int argc, char **argv) simple_string_list_append(&extension_exclude_patterns, optarg); break; - + case 18: + statistics_only = true; + break; + case 19: + no_data = true; + break; + case 20: + no_schema = true; + break; + case 21: + no_statistics = true; + break; case 25: dopt.restrict_key = pg_strdup(optarg); break; - default: /* getopt_long already emitted a complaint */ pg_log_error_hint("Try \"%s --help\" for more information.", progname); @@ -734,6 +752,17 @@ main(int argc, char **argv) if (data_only && schema_only) pg_fatal("options -s/--schema-only and -a/--data-only cannot be used together"); + if (schema_only && statistics_only) + pg_fatal("options -s/--schema-only and --statistics-only cannot be used together"); + if (data_only && statistics_only) + pg_fatal("options -a/--data-only and --statistics-only cannot be used together"); + + if (data_only && no_data) + pg_fatal("options -a/--data-only and --no-data cannot be used together"); + if (schema_only && no_schema) + pg_fatal("options -s/--schema-only and --no-schema cannot be used together"); + if (statistics_only && no_statistics) + pg_fatal("options --statistics-only and --no-statistics cannot be used together"); if (schema_only && foreign_servers_include_patterns.head != NULL) pg_fatal("options -s/--schema-only and --include-foreign-data cannot be used together"); @@ -748,8 +777,9 @@ main(int argc, char **argv) pg_fatal("option --if-exists requires option -c/--clean"); /* set derivative flags */ - dopt.dumpSchema = (!data_only); - dopt.dumpData = (!schema_only); + dopt.dumpData = data_only || (!schema_only && !statistics_only && !no_data); + dopt.dumpSchema = schema_only || (!data_only && !statistics_only && !no_schema); + dopt.dumpStatistics = statistics_only || (!data_only && !schema_only && !no_statistics); /* * --inserts are already implied above if --column-inserts or @@ -1056,6 +1086,7 @@ main(int argc, char **argv) ropt->dropSchema = dopt.outputClean; ropt->dumpData = dopt.dumpData; ropt->dumpSchema = dopt.dumpSchema; + ropt->dumpStatistics = dopt.dumpStatistics; ropt->if_exists = dopt.if_exists; ropt->column_inserts = dopt.column_inserts; ropt->dumpSections = dopt.dumpSections; @@ -1135,7 +1166,7 @@ help(const char *progname) printf(_(" -?, --help show this help, then exit\n")); printf(_("\nOptions controlling the output content:\n")); - printf(_(" -a, --data-only dump only the data, not the schema\n")); + printf(_(" -a, --data-only dump only the data, not the schema or statistics\n")); printf(_(" -b, --large-objects include large objects in dump\n")); printf(_(" --blobs (same as --large-objects, deprecated)\n")); printf(_(" -B, --no-large-objects exclude large objects in dump\n")); @@ -1148,7 +1179,7 @@ help(const char *progname) printf(_(" -N, --exclude-schema=PATTERN do NOT dump the specified schema(s)\n")); printf(_(" -O, --no-owner skip restoration of object ownership in\n" " plain-text format\n")); - printf(_(" -s, --schema-only dump only the schema, no data\n")); + printf(_(" -s, --schema-only dump only the schema, no data or statistics\n")); printf(_(" -S, --superuser=NAME superuser user name to use in plain-text format\n")); printf(_(" -t, --table=PATTERN dump only the specified table(s)\n")); printf(_(" -T, --exclude-table=PATTERN do NOT dump the specified table(s)\n")); @@ -1178,7 +1209,9 @@ help(const char *progname) printf(_(" --load-via-partition-root load partitions via the root table\n")); printf(_(" --no-comments do not dump comments\n")); printf(_(" --no-publications do not dump publications\n")); + printf(_(" --no-schema do not dump schema\n")); printf(_(" --no-security-labels do not dump security label assignments\n")); + printf(_(" --no-statistics do not dump statistics\n")); printf(_(" --no-subscriptions do not dump subscriptions\n")); printf(_(" --no-table-access-method do not dump table access methods\n")); printf(_(" --no-tablespaces do not dump tablespace assignments\n")); @@ -1191,6 +1224,7 @@ help(const char *progname) printf(_(" --section=SECTION dump named section (pre-data, data, or post-data)\n")); printf(_(" --serializable-deferrable wait until the dump can run without anomalies\n")); printf(_(" --snapshot=SNAPSHOT use given snapshot for the dump\n")); + printf(_(" --statistics-only dump only the statistics, not schema or data\n")); printf(_(" --strict-names require table and/or schema include patterns to\n" " match at least one entity each\n")); printf(_(" --table-and-children=PATTERN dump only the specified table(s), including\n" @@ -6801,6 +6835,45 @@ getFuncs(Archive *fout, int *numFuncs) return finfo; } +/* + * getRelationStatistics + * register the statistics object as a dependent of the relation. + * + */ +static RelStatsInfo * +getRelationStatistics(Archive *fout, DumpableObject *rel, char relkind) +{ + if (!fout->dopt->dumpStatistics) + return NULL; + + if ((relkind == RELKIND_RELATION) || + (relkind == RELKIND_PARTITIONED_TABLE) || + (relkind == RELKIND_INDEX) || + (relkind == RELKIND_PARTITIONED_INDEX) || + (relkind == RELKIND_MATVIEW)) + { + RelStatsInfo *info = pg_malloc0(sizeof(RelStatsInfo)); + DumpableObject *dobj = &info->dobj; + + dobj->objType = DO_REL_STATS; + dobj->catId.tableoid = 0; + dobj->catId.oid = 0; + AssignDumpId(dobj); + dobj->dependencies = (DumpId *) pg_malloc(sizeof(DumpId)); + dobj->dependencies[0] = rel->dumpId; + dobj->nDeps = 1; + dobj->allocDeps = 1; + dobj->components |= DUMP_COMPONENT_STATISTICS; + dobj->name = pg_strdup(rel->name); + dobj->namespace = rel->namespace; + info->relkind = relkind; + info->postponed_def = false; + + return info; + } + return NULL; +} + /* * getTables * read all the tables (no indexes) in the system catalogs, @@ -7160,8 +7233,8 @@ getTables(Archive *fout, int *numTables) /* * Now, consider the table "interesting" if we need to dump its - * definition or its data. Later on, we'll skip a lot of data - * collection for uninteresting tables. + * definition, data or its statistics. Later on, we'll skip a lot of + * data collection for uninteresting tables. * * Note: the "interesting" flag will also be set by flagInhTables for * parents of interesting tables, so that we collect necessary @@ -7171,7 +7244,8 @@ getTables(Archive *fout, int *numTables) */ tblinfo[i].interesting = (tblinfo[i].dobj.dump & (DUMP_COMPONENT_DEFINITION | - DUMP_COMPONENT_DATA)) != 0; + DUMP_COMPONENT_DATA | + DUMP_COMPONENT_STATISTICS)) != 0; tblinfo[i].dummy_view = false; /* might get set during sort */ tblinfo[i].postponed_def = false; /* might get set during sort */ @@ -7184,6 +7258,10 @@ getTables(Archive *fout, int *numTables) tblinfo[i].dobj.components |= DUMP_COMPONENT_ACL; tblinfo[i].hascolumnACLs = false; /* may get set later */ + /* Add statistics */ + if (tblinfo[i].interesting) + getRelationStatistics(fout, &tblinfo[i].dobj, tblinfo[i].relkind); + /* * Read-lock target tables to make sure they aren't DROPPED or altered * in schema before we get around to dumping them. @@ -7663,6 +7741,8 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables) for (int c = 0; c < numinds; c++, j++) { char contype; + char indexkind; + RelStatsInfo *relstats; indxinfo[j].dobj.objType = DO_INDEX; indxinfo[j].dobj.catId.tableoid = atooid(PQgetvalue(res, j, i_tableoid)); @@ -7690,7 +7770,14 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables) { NULL, NULL }; + + if (indxinfo[j].parentidx == 0) + indexkind = RELKIND_INDEX; + else + indexkind = RELKIND_PARTITIONED_INDEX; + contype = *(PQgetvalue(res, j, i_contype)); + relstats = getRelationStatistics(fout, &indxinfo[j].dobj, indexkind); if (contype == 'p' || contype == 'u' || contype == 'x') { @@ -7723,6 +7810,8 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables) constrinfo->separate = true; indxinfo[j].indexconstraint = constrinfo->dobj.dumpId; + if (relstats != NULL) + addObjectDependency(&relstats->dobj, constrinfo->dobj.dumpId); } else { @@ -10258,6 +10347,296 @@ dumpComment(Archive *fout, const char *type, catalogId, subid, dumpId, NULL); } +/* + * Tabular description of the parameters to pg_restore_relation_stats() + * param_name, param_type + */ +static const char *rel_stats_arginfo[][2] = { + {"relation", "regclass"}, + {"version", "integer"}, + {"relpages", "integer"}, + {"reltuples", "real"}, + {"relallvisible", "integer"}, +}; + +/* + * Tabular description of the parameters to pg_restore_attribute_stats() + * param_name, param_type + */ +static const char *att_stats_arginfo[][2] = { + {"relation", "regclass"}, + {"attname", "name"}, + {"inherited", "boolean"}, + {"version", "integer"}, + {"null_frac", "float4"}, + {"avg_width", "integer"}, + {"n_distinct", "float4"}, + {"most_common_vals", "text"}, + {"most_common_freqs", "float4[]"}, + {"histogram_bounds", "text"}, + {"correlation", "float4"}, + {"most_common_elems", "text"}, + {"most_common_elem_freqs", "float4[]"}, + {"elem_count_histogram", "float4[]"}, + {"range_length_histogram", "text"}, + {"range_empty_frac", "float4"}, + {"range_bounds_histogram", "text"}, +}; + +/* + * getRelStatsExportQuery -- + * + * Generate a query that will fetch all relation (e.g. pg_class) + * stats for a given relation. + */ +static void +getRelStatsExportQuery(PQExpBuffer query, Archive *fout, + const char *schemaname, const char *relname) +{ + resetPQExpBuffer(query); + appendPQExpBufferStr(query, + "SELECT c.oid::regclass AS relation, " + "current_setting('server_version_num') AS version, " + "c.relpages, c.reltuples, c.relallvisible " + "FROM pg_class c " + "JOIN pg_namespace n " + "ON n.oid = c.relnamespace " + "WHERE n.nspname = "); + appendStringLiteralAH(query, schemaname, fout); + appendPQExpBufferStr(query, " AND c.relname = "); + appendStringLiteralAH(query, relname, fout); +} + +/* + * getAttStatsExportQuery -- + * + * Generate a query that will fetch all attribute (e.g. pg_statistic) + * stats for a given relation. + */ +static void +getAttStatsExportQuery(PQExpBuffer query, Archive *fout, + const char *schemaname, const char *relname) +{ + resetPQExpBuffer(query); + appendPQExpBufferStr(query, + "SELECT c.oid::regclass AS relation, " + "s.attname," + "s.inherited," + "current_setting('server_version_num') AS version, " + "s.null_frac," + "s.avg_width," + "s.n_distinct," + "s.most_common_vals," + "s.most_common_freqs," + "s.histogram_bounds," + "s.correlation," + "s.most_common_elems," + "s.most_common_elem_freqs," + "s.elem_count_histogram,"); + + if (fout->remoteVersion >= 170000) + appendPQExpBufferStr(query, + "s.range_length_histogram," + "s.range_empty_frac," + "s.range_bounds_histogram "); + else + appendPQExpBufferStr(query, + "NULL AS range_length_histogram," + "NULL AS range_empty_frac," + "NULL AS range_bounds_histogram "); + + appendPQExpBufferStr(query, + "FROM pg_stats s " + "JOIN pg_namespace n " + "ON n.nspname = s.schemaname " + "JOIN pg_class c " + "ON c.relname = s.tablename " + "AND c.relnamespace = n.oid " + "WHERE s.schemaname = "); + appendStringLiteralAH(query, schemaname, fout); + appendPQExpBufferStr(query, " AND s.tablename = "); + appendStringLiteralAH(query, relname, fout); + appendPQExpBufferStr(query, " ORDER BY s.attname, s.inherited"); +} + + +/* + * appendNamedArgument -- + * + * Convenience routine for constructing parameters of the form: + * 'paraname', 'value'::type + */ +static void +appendNamedArgument(PQExpBuffer out, Archive *fout, const char *argname, + const char *argval, const char *argtype) +{ + appendPQExpBufferStr(out, "\t"); + + appendStringLiteralAH(out, argname, fout); + appendPQExpBufferStr(out, ", "); + + appendStringLiteralAH(out, argval, fout); + appendPQExpBuffer(out, "::%s", argtype); +} + +/* + * appendRelStatsImport -- + * + * Append a formatted pg_restore_relation_stats statement. + */ +static void +appendRelStatsImport(PQExpBuffer out, Archive *fout, PGresult *res) +{ + const char *sep = ""; + + if (PQntuples(res) == 0) + return; + + appendPQExpBufferStr(out, "SELECT * FROM pg_catalog.pg_restore_relation_stats(\n"); + + for (int argno = 0; argno < lengthof(rel_stats_arginfo); argno++) + { + const char *argname = rel_stats_arginfo[argno][0]; + const char *argtype = rel_stats_arginfo[argno][1]; + int fieldno = PQfnumber(res, argname); + + if (fieldno < 0) + pg_fatal("relation stats export query missing field '%s'", + argname); + + if (PQgetisnull(res, 0, fieldno)) + continue; + + appendPQExpBufferStr(out, sep); + appendNamedArgument(out, fout, argname, PQgetvalue(res, 0, fieldno), argtype); + + sep = ",\n"; + } + appendPQExpBufferStr(out, "\n);\n"); +} + +/* + * appendAttStatsImport -- + * + * Append a series of formatted pg_restore_attribute_stats statements. + */ +static void +appendAttStatsImport(PQExpBuffer out, Archive *fout, PGresult *res) +{ + for (int rownum = 0; rownum < PQntuples(res); rownum++) + { + const char *sep = ""; + + appendPQExpBufferStr(out, "SELECT * FROM pg_catalog.pg_restore_attribute_stats(\n"); + for (int argno = 0; argno < lengthof(att_stats_arginfo); argno++) + { + const char *argname = att_stats_arginfo[argno][0]; + const char *argtype = att_stats_arginfo[argno][1]; + int fieldno = PQfnumber(res, argname); + + if (fieldno < 0) + pg_fatal("attribute stats export query missing field '%s'", + argname); + + if (PQgetisnull(res, rownum, fieldno)) + continue; + + appendPQExpBufferStr(out, sep); + appendNamedArgument(out, fout, argname, PQgetvalue(res, rownum, fieldno), argtype); + sep = ",\n"; + } + appendPQExpBufferStr(out, "\n);\n"); + } +} + +/* + * Decide which section to use based on the relkind of the parent object. + * + * NB: materialized views may be postponed from SECTION_PRE_DATA to + * SECTION_POST_DATA to resolve some kinds of dependency problems. If so, the + * matview stats will also be postponed to SECTION_POST_DATA. See + * repairMatViewBoundaryMultiLoop(). + */ +static teSection +statisticsDumpSection(const RelStatsInfo *rsinfo) +{ + switch (rsinfo->relkind) + { + case RELKIND_RELATION: + case RELKIND_PARTITIONED_TABLE: + case RELKIND_MATVIEW: + return SECTION_DATA; + case RELKIND_INDEX: + case RELKIND_PARTITIONED_INDEX: + return SECTION_POST_DATA; + default: + pg_fatal("cannot dump statistics for relation kind '%c'", + rsinfo->relkind); + } + + return 0; /* keep compiler quiet */ +} + +/* + * dumpRelationStats -- + * + * Dump command to import stats into the relation on the new database. + */ +static void +dumpRelationStats(Archive *fout, const RelStatsInfo *rsinfo) +{ + PGresult *res; + PQExpBuffer query; + PQExpBuffer out; + PQExpBuffer tag; + DumpableObject *dobj = (DumpableObject *) &rsinfo->dobj; + DumpId *deps = NULL; + int ndeps = 0; + + /* nothing to do if we are not dumping statistics */ + if (!fout->dopt->dumpStatistics) + return; + + /* dependent on the relation definition, if doing schema */ + if (fout->dopt->dumpSchema) + { + deps = dobj->dependencies; + ndeps = dobj->nDeps; + } + + tag = createPQExpBuffer(); + appendPQExpBufferStr(tag, fmtId(dobj->name)); + + query = createPQExpBuffer(); + out = createPQExpBuffer(); + + getRelStatsExportQuery(query, fout, dobj->namespace->dobj.name, + dobj->name); + res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK); + appendRelStatsImport(out, fout, res); + PQclear(res); + + getAttStatsExportQuery(query, fout, dobj->namespace->dobj.name, + dobj->name); + res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK); + appendAttStatsImport(out, fout, res); + PQclear(res); + + ArchiveEntry(fout, nilCatalogId, createDumpId(), + ARCHIVE_OPTS(.tag = tag->data, + .namespace = dobj->namespace->dobj.name, + .description = "STATISTICS DATA", + .section = rsinfo->postponed_def ? + SECTION_POST_DATA : statisticsDumpSection(rsinfo), + .createStmt = out->data, + .deps = deps, + .nDeps = ndeps)); + + destroyPQExpBuffer(query); + destroyPQExpBuffer(out); + destroyPQExpBuffer(tag); +} + /* * dumpTableComment -- * @@ -10706,6 +11085,9 @@ dumpDumpableObject(Archive *fout, DumpableObject *dobj) case DO_SUBSCRIPTION_REL: dumpSubscriptionTable(fout, (const SubRelInfo *) dobj); break; + case DO_REL_STATS: + dumpRelationStats(fout, (const RelStatsInfo *) dobj); + break; case DO_PRE_DATA_BOUNDARY: case DO_POST_DATA_BOUNDARY: /* never dumped, nothing to do */ @@ -18817,6 +19199,16 @@ addBoundaryDependencies(DumpableObject **dobjs, int numObjs, /* must come after the pre-data boundary */ addObjectDependency(dobj, preDataBound->dumpId); break; + case DO_REL_STATS: + /* stats section varies by parent object type, DATA or POST */ + if (statisticsDumpSection((RelStatsInfo *) dobj) == SECTION_DATA) + { + addObjectDependency(dobj, preDataBound->dumpId); + addObjectDependency(postDataBound, dobj->dumpId); + } + else + addObjectDependency(dobj, postDataBound->dumpId); + break; } } } diff --git a/src/bin/pg_dump/pg_dump.h b/src/bin/pg_dump/pg_dump.h index 2de5afdacdb..08b5e49ee3c 100644 --- a/src/bin/pg_dump/pg_dump.h +++ b/src/bin/pg_dump/pg_dump.h @@ -82,10 +82,13 @@ typedef enum DO_PUBLICATION, DO_PUBLICATION_REL, DO_PUBLICATION_TABLE_IN_SCHEMA, + DO_REL_STATS, DO_SUBSCRIPTION, DO_SUBSCRIPTION_REL, /* see note for SubRelInfo */ } DumpableObjectType; +#define NUM_DUMPABLE_OBJECT_TYPES (DO_SUBSCRIPTION_REL + 1) + /* * DumpComponents is a bitmask of the potentially dumpable components of * a database object: its core definition, plus optional attributes such @@ -109,6 +112,7 @@ typedef uint32 DumpComponents; #define DUMP_COMPONENT_ACL (1 << 4) #define DUMP_COMPONENT_POLICY (1 << 5) #define DUMP_COMPONENT_USERMAP (1 << 6) +#define DUMP_COMPONENT_STATISTICS (1 << 7) #define DUMP_COMPONENT_ALL (0xFFFF) /* @@ -136,6 +140,7 @@ typedef uint32 DumpComponents; #define DUMP_COMPONENTS_REQUIRING_LOCK (\ DUMP_COMPONENT_DEFINITION |\ DUMP_COMPONENT_DATA |\ + DUMP_COMPONENT_STATISTICS |\ DUMP_COMPONENT_POLICY) typedef struct _dumpableObject @@ -431,6 +436,13 @@ typedef struct _indexAttachInfo IndxInfo *partitionIdx; /* link to index on partition */ } IndexAttachInfo; +typedef struct _relStatsInfo +{ + DumpableObject dobj; + char relkind; /* 'r', 'm', 'i', etc */ + bool postponed_def; /* stats must be postponed into post-data */ +} RelStatsInfo; + typedef struct _statsExtInfo { DumpableObject dobj; diff --git a/src/bin/pg_dump/pg_dump_sort.c b/src/bin/pg_dump/pg_dump_sort.c index 7618c1bdf1c..59f7e4f92bb 100644 --- a/src/bin/pg_dump/pg_dump_sort.c +++ b/src/bin/pg_dump/pg_dump_sort.c @@ -82,6 +82,7 @@ enum dbObjectTypePriorities PRIO_SEQUENCE_SET, PRIO_LARGE_OBJECT, PRIO_LARGE_OBJECT_DATA, + PRIO_STATISTICS_DATA_DATA, PRIO_POST_DATA_BOUNDARY, /* boundary! */ PRIO_CONSTRAINT, PRIO_INDEX, @@ -149,11 +150,12 @@ static const int dbObjectTypePriority[] = [DO_PUBLICATION] = PRIO_PUBLICATION, [DO_PUBLICATION_REL] = PRIO_PUBLICATION_REL, [DO_PUBLICATION_TABLE_IN_SCHEMA] = PRIO_PUBLICATION_TABLE_IN_SCHEMA, + [DO_REL_STATS] = PRIO_STATISTICS_DATA_DATA, [DO_SUBSCRIPTION] = PRIO_SUBSCRIPTION, [DO_SUBSCRIPTION_REL] = PRIO_SUBSCRIPTION_REL, }; -StaticAssertDecl(lengthof(dbObjectTypePriority) == (DO_SUBSCRIPTION_REL + 1), +StaticAssertDecl(lengthof(dbObjectTypePriority) == NUM_DUMPABLE_OBJECT_TYPES, "array length mismatch"); static DumpId preDataBoundId; @@ -1026,11 +1028,22 @@ repairMatViewBoundaryMultiLoop(DumpableObject *boundaryobj, { /* remove boundary's dependency on object after it in loop */ removeObjectDependency(boundaryobj, nextobj->dumpId); - /* if that object is a matview, mark it as postponed into post-data */ + + /* + * If that object is a matview or matview stats, mark it as postponed into + * post-data. + */ if (nextobj->objType == DO_TABLE) { TableInfo *nextinfo = (TableInfo *) nextobj; + if (nextinfo->relkind == RELKIND_MATVIEW) + nextinfo->postponed_def = true; + } + else if (nextobj->objType == DO_REL_STATS) + { + RelStatsInfo *nextinfo = (RelStatsInfo *) nextobj; + if (nextinfo->relkind == RELKIND_MATVIEW) nextinfo->postponed_def = true; } @@ -1243,6 +1256,21 @@ repairDependencyLoop(DumpableObject **loop, { DumpableObject *nextobj; + nextobj = (j < nLoop - 1) ? loop[j + 1] : loop[0]; + repairMatViewBoundaryMultiLoop(loop[j], nextobj); + return; + } + } + } + else if (loop[i]->objType == DO_REL_STATS && + ((RelStatsInfo *) loop[i])->relkind == RELKIND_MATVIEW) + { + for (j = 0; j < nLoop; j++) + { + if (loop[j]->objType == DO_POST_DATA_BOUNDARY) + { + DumpableObject *nextobj; + nextobj = (j < nLoop - 1) ? loop[j + 1] : loop[0]; repairMatViewBoundaryMultiLoop(loop[j], nextobj); return; @@ -1728,6 +1756,11 @@ describeDumpableObject(DumpableObject *obj, char *buf, int bufsize) "POST-DATA BOUNDARY (ID %d)", obj->dumpId); return; + case DO_REL_STATS: + snprintf(buf, bufsize, + "RELATION STATISTICS FOR %s (ID %d OID %u)", + obj->name, obj->dumpId, obj->catId.oid); + return; } /* shouldn't get here */ snprintf(buf, bufsize, diff --git a/src/bin/pg_dump/pg_dumpall.c b/src/bin/pg_dump/pg_dumpall.c index 0a0bfa9bbbb..22af86d5b1f 100644 --- a/src/bin/pg_dump/pg_dumpall.c +++ b/src/bin/pg_dump/pg_dumpall.c @@ -103,6 +103,9 @@ static int use_setsessauth = 0; static int no_comments = 0; static int no_publications = 0; static int no_security_labels = 0; +static int no_data = 0; +static int no_schema = 0; +static int no_statistics = 0; static int no_subscriptions = 0; static int no_toast_compression = 0; static int no_unlogged_table_data = 0; @@ -110,6 +113,7 @@ static int no_role_passwords = 0; static int server_version; static int load_via_partition_root = 0; static int on_conflict_do_nothing = 0; +static int statistics_only = 0; static char role_catalog[10]; #define PG_AUTHID "pg_authid" @@ -170,15 +174,19 @@ main(int argc, char *argv[]) {"role", required_argument, NULL, 3}, {"use-set-session-authorization", no_argument, &use_setsessauth, 1}, {"no-comments", no_argument, &no_comments, 1}, + {"no-data", no_argument, &no_data, 1}, {"no-publications", no_argument, &no_publications, 1}, {"no-role-passwords", no_argument, &no_role_passwords, 1}, + {"no-schema", no_argument, &no_schema, 1}, {"no-security-labels", no_argument, &no_security_labels, 1}, {"no-subscriptions", no_argument, &no_subscriptions, 1}, + {"no-statistics", no_argument, &no_statistics, 1}, {"no-sync", no_argument, NULL, 4}, {"no-toast-compression", no_argument, &no_toast_compression, 1}, {"no-unlogged-table-data", no_argument, &no_unlogged_table_data, 1}, {"on-conflict-do-nothing", no_argument, &on_conflict_do_nothing, 1}, {"rows-per-insert", required_argument, NULL, 7}, + {"statistics-only", no_argument, &statistics_only, 1}, {"filter", required_argument, NULL, 8}, {"restrict-key", required_argument, NULL, 9}, @@ -456,10 +464,16 @@ main(int argc, char *argv[]) appendPQExpBufferStr(pgdumpopts, " --use-set-session-authorization"); if (no_comments) appendPQExpBufferStr(pgdumpopts, " --no-comments"); + if (no_data) + appendPQExpBufferStr(pgdumpopts, " --no-data"); if (no_publications) appendPQExpBufferStr(pgdumpopts, " --no-publications"); if (no_security_labels) appendPQExpBufferStr(pgdumpopts, " --no-security-labels"); + if (no_schema) + appendPQExpBufferStr(pgdumpopts, " --no-schema"); + if (no_statistics) + appendPQExpBufferStr(pgdumpopts, " --no-statistics"); if (no_subscriptions) appendPQExpBufferStr(pgdumpopts, " --no-subscriptions"); if (no_toast_compression) @@ -468,6 +482,8 @@ main(int argc, char *argv[]) appendPQExpBufferStr(pgdumpopts, " --no-unlogged-table-data"); if (on_conflict_do_nothing) appendPQExpBufferStr(pgdumpopts, " --on-conflict-do-nothing"); + if (statistics_only) + appendPQExpBufferStr(pgdumpopts, " --statistics-only"); /* * If you don't provide a restrict key, one will be appointed for you. @@ -678,13 +694,13 @@ help(void) printf(_(" --lock-wait-timeout=TIMEOUT fail after waiting TIMEOUT for a table lock\n")); printf(_(" -?, --help show this help, then exit\n")); printf(_("\nOptions controlling the output content:\n")); - printf(_(" -a, --data-only dump only the data, not the schema\n")); + printf(_(" -a, --data-only dump only the data, not the schema or statistics\n")); printf(_(" -c, --clean clean (drop) databases before recreating\n")); printf(_(" -E, --encoding=ENCODING dump the data in encoding ENCODING\n")); printf(_(" -g, --globals-only dump only global objects, no databases\n")); printf(_(" -O, --no-owner skip restoration of object ownership\n")); printf(_(" -r, --roles-only dump only roles, no databases or tablespaces\n")); - printf(_(" -s, --schema-only dump only the schema, no data\n")); + printf(_(" -s, --schema-only dump only the schema, no data or statistics\n")); printf(_(" -S, --superuser=NAME superuser user name to use in the dump\n")); printf(_(" -t, --tablespaces-only dump only tablespaces, no databases or roles\n")); printf(_(" -x, --no-privileges do not dump privileges (grant/revoke)\n")); @@ -701,7 +717,9 @@ help(void) printf(_(" --no-comments do not dump comments\n")); printf(_(" --no-publications do not dump publications\n")); printf(_(" --no-role-passwords do not dump passwords for roles\n")); + printf(_(" --no-schema do not dump schema\n")); printf(_(" --no-security-labels do not dump security label assignments\n")); + printf(_(" --no-statistics do not dump statistics\n")); printf(_(" --no-subscriptions do not dump subscriptions\n")); printf(_(" --no-sync do not wait for changes to be written safely to disk\n")); printf(_(" --no-table-access-method do not dump table access methods\n")); @@ -712,6 +730,7 @@ help(void) printf(_(" --quote-all-identifiers quote all identifiers, even if not key words\n")); printf(_(" --restrict-key=RESTRICT_KEY use provided string as psql \\restrict key\n")); printf(_(" --rows-per-insert=NROWS number of rows per INSERT; implies --inserts\n")); + printf(_(" --statistics-only dump only the statistics, not schema or data\n")); printf(_(" --use-set-session-authorization\n" " use SET SESSION AUTHORIZATION commands instead of\n" " ALTER OWNER commands to set ownership\n")); diff --git a/src/bin/pg_dump/pg_restore.c b/src/bin/pg_dump/pg_restore.c index ae9d20cfe82..928e56f8564 100644 --- a/src/bin/pg_dump/pg_restore.c +++ b/src/bin/pg_dump/pg_restore.c @@ -64,6 +64,8 @@ main(int argc, char **argv) int numWorkers = 1; Archive *AH; char *inputFileSpec; + bool data_only = false; + bool schema_only = false; static int disable_triggers = 0; static int enable_row_security = 0; static int if_exists = 0; @@ -72,12 +74,14 @@ main(int argc, char **argv) static int outputNoTablespaces = 0; static int use_setsessauth = 0; static int no_comments = 0; + static int no_data = 0; static int no_publications = 0; + static int no_schema = 0; static int no_security_labels = 0; + static int no_statistics = 0; static int no_subscriptions = 0; static int strict_names = 0; - bool data_only = false; - bool schema_only = false; + static int statistics_only = 0; struct option cmdopts[] = { {"clean", 0, NULL, 'c'}, @@ -125,9 +129,13 @@ main(int argc, char **argv) {"transaction-size", required_argument, NULL, 5}, {"use-set-session-authorization", no_argument, &use_setsessauth, 1}, {"no-comments", no_argument, &no_comments, 1}, + {"no-data", no_argument, &no_data, 1}, {"no-publications", no_argument, &no_publications, 1}, + {"no-schema", no_argument, &no_schema, 1}, {"no-security-labels", no_argument, &no_security_labels, 1}, {"no-subscriptions", no_argument, &no_subscriptions, 1}, + {"no-statistics", no_argument, &no_statistics, 1}, + {"statistics-only", no_argument, &statistics_only, 1}, {"filter", required_argument, NULL, 4}, {"restrict-key", required_argument, NULL, 6}, @@ -365,6 +373,10 @@ main(int argc, char **argv) if (data_only && schema_only) pg_fatal("options -s/--schema-only and -a/--data-only cannot be used together"); + if (data_only && statistics_only) + pg_fatal("options -a/--data-only and --statistics-only cannot be used together"); + if (schema_only && statistics_only) + pg_fatal("options -s/--schema-only and --statistics-only cannot be used together"); if (data_only && opts->dropSchema) pg_fatal("options -c/--clean and -a/--data-only cannot be used together"); @@ -384,8 +396,9 @@ main(int argc, char **argv) pg_fatal("cannot specify both --single-transaction and multiple jobs"); /* set derivative flags */ - opts->dumpSchema = (!data_only); - opts->dumpData = (!schema_only); + opts->dumpData = data_only || (!no_data && !schema_only && !statistics_only); + opts->dumpSchema = schema_only || (!no_schema && !data_only && !statistics_only); + opts->dumpStatistics = statistics_only || (!no_statistics && !data_only && !schema_only); opts->disable_triggers = disable_triggers; opts->enable_row_security = enable_row_security; @@ -516,12 +529,15 @@ usage(const char *progname) printf(_(" --no-data-for-failed-tables do not restore data of tables that could not be\n" " created\n")); printf(_(" --no-publications do not restore publications\n")); + printf(_(" --no-schema do not restore schema\n")); printf(_(" --no-security-labels do not restore security labels\n")); + printf(_(" --no-statistics do not restore statistics\n")); printf(_(" --no-subscriptions do not restore subscriptions\n")); printf(_(" --no-table-access-method do not restore table access methods\n")); printf(_(" --no-tablespaces do not restore tablespace assignments\n")); printf(_(" --restrict-key=RESTRICT_KEY use provided string as psql \\restrict key\n")); printf(_(" --section=SECTION restore named section (pre-data, data, or post-data)\n")); + printf(_(" --statistics-only restore only the statistics, not schema or data\n")); printf(_(" --strict-names require table and/or schema include patterns to\n" " match at least one entity each\n")); printf(_(" --transaction-size=N commit after every N objects\n")); diff --git a/src/bin/pg_dump/t/001_basic.pl b/src/bin/pg_dump/t/001_basic.pl index b9d13a0e1de..56b50173e7a 100644 --- a/src/bin/pg_dump/t/001_basic.pl +++ b/src/bin/pg_dump/t/001_basic.pl @@ -50,12 +50,30 @@ 'pg_dump: options -s/--schema-only and -a/--data-only cannot be used together' ); +command_fails_like( + [ 'pg_dump', '-s', '--statistics-only' ], + qr/\Qpg_dump: error: options -s\/--schema-only and --statistics-only cannot be used together\E/, + 'pg_dump: error: options -s/--schema-only and --statistics-only cannot be used together' +); + +command_fails_like( + [ 'pg_dump', '-a', '--statistics-only' ], + qr/\Qpg_dump: error: options -a\/--data-only and --statistics-only cannot be used together\E/, + 'pg_dump: error: options -a/--data-only and --statistics-only cannot be used together' +); + command_fails_like( [ 'pg_dump', '-s', '--include-foreign-data=xxx' ], qr/\Qpg_dump: error: options -s\/--schema-only and --include-foreign-data cannot be used together\E/, 'pg_dump: options -s/--schema-only and --include-foreign-data cannot be used together' ); +command_fails_like( + [ 'pg_dump', '--statistics-only', '--no-statistics' ], + qr/\Qpg_dump: error: options --statistics-only and --no-statistics cannot be used together\E/, + 'pg_dump: options --statistics-only and --no-statistics cannot be used together' +); + command_fails_like( [ 'pg_dump', '-j2', '--include-foreign-data=xxx' ], qr/\Qpg_dump: error: option --include-foreign-data is not supported with parallel backup\E/, diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl index 47dc420e61f..1d68b4d5161 100644 --- a/src/bin/pg_dump/t/002_pg_dump.pl +++ b/src/bin/pg_dump/t/002_pg_dump.pl @@ -66,7 +66,7 @@ '--format=custom', "--file=$tempdir/binary_upgrade.dump", '-w', - '--schema-only', + '--no-data', '--binary-upgrade', '-d', 'postgres', # alternative way to specify database ], @@ -677,6 +677,34 @@ '--schema=dump_test', '-b', '-B', '--no-sync', 'postgres', ], + }, + no_statistics => { + dump_cmd => [ + 'pg_dump', '--no-sync', + "--file=$tempdir/no_statistics.sql", '--no-statistics', + 'postgres', + ], + }, + no_data_no_schema => { + dump_cmd => [ + 'pg_dump', '--no-sync', + "--file=$tempdir/no_data_no_schema.sql", '--no-data', + '--no-schema', 'postgres', + ], + }, + statistics_only => { + dump_cmd => [ + 'pg_dump', '--no-sync', + "--file=$tempdir/statistics_only.sql", '--statistics-only', + 'postgres', + ], + }, + no_schema => { + dump_cmd => [ + 'pg_dump', '--no-sync', + "--file=$tempdir/no_schema.sql", '--no-schema', + 'postgres', + ], },); ############################################################### @@ -745,6 +773,7 @@ no_privs => 1, no_subscriptions => 1, no_subscriptions_restore => 1, + no_statistics => 1, no_table_access_method => 1, pg_dumpall_dbprivs => 1, pg_dumpall_exclude => 1, @@ -956,6 +985,7 @@ column_inserts => 1, data_only => 1, inserts => 1, + no_schema => 1, section_data => 1, test_schema_plus_large_objects => 1, }, @@ -1333,6 +1363,7 @@ column_inserts => 1, data_only => 1, inserts => 1, + no_schema => 1, section_data => 1, test_schema_plus_large_objects => 1, }, @@ -1354,6 +1385,7 @@ column_inserts => 1, data_only => 1, inserts => 1, + no_schema => 1, section_data => 1, test_schema_plus_large_objects => 1, }, @@ -1375,6 +1407,7 @@ column_inserts => 1, data_only => 1, inserts => 1, + no_schema => 1, section_data => 1, test_schema_plus_large_objects => 1, }, @@ -1541,6 +1574,7 @@ column_inserts => 1, data_only => 1, inserts => 1, + no_schema => 1, section_data => 1, test_schema_plus_large_objects => 1, }, @@ -1698,6 +1732,7 @@ %full_runs, %dump_test_schema_runs, data_only => 1, + no_schema => 1, only_dump_test_table => 1, section_data => 1, }, @@ -1725,6 +1760,7 @@ data_only => 1, exclude_test_table => 1, exclude_test_table_data => 1, + no_schema => 1, section_data => 1, }, unlike => { @@ -1745,7 +1781,10 @@ \QCOPY dump_test.fk_reference_test_table (col1) FROM stdin;\E \n(?:\d\n){5}\\\.\n /xms, - like => { data_only => 1, }, + like => { + data_only => 1, + no_schema => 1, + }, }, 'COPY test_second_table' => { @@ -1761,6 +1800,7 @@ %full_runs, %dump_test_schema_runs, data_only => 1, + no_schema => 1, section_data => 1, }, unlike => { @@ -1783,6 +1823,7 @@ %full_runs, %dump_test_schema_runs, data_only => 1, + no_schema => 1, section_data => 1, }, unlike => { @@ -1806,6 +1847,7 @@ %full_runs, %dump_test_schema_runs, data_only => 1, + no_schema => 1, section_data => 1, }, unlike => { @@ -1828,6 +1870,7 @@ %full_runs, %dump_test_schema_runs, data_only => 1, + no_schema => 1, section_data => 1, }, unlike => { @@ -1850,6 +1893,7 @@ %full_runs, %dump_test_schema_runs, data_only => 1, + no_schema => 1, section_data => 1, }, unlike => { @@ -3295,6 +3339,7 @@ like => { %full_runs, data_only => 1, + no_schema => 1, section_data => 1, only_dump_test_schema => 1, test_schema_plus_large_objects => 1, @@ -3465,6 +3510,7 @@ %full_runs, %dump_test_schema_runs, data_only => 1, + no_schema => 1, only_dump_measurement => 1, section_data => 1, only_dump_test_schema => 1, @@ -4345,6 +4391,7 @@ column_inserts => 1, data_only => 1, inserts => 1, + no_schema => 1, section_data => 1, test_schema_plus_large_objects => 1, binary_upgrade => 1, @@ -4645,6 +4692,61 @@ }, }, + # + # TABLE and MATVIEW stats will end up in SECTION_DATA. + # INDEX stats (expression columns only) will end up in SECTION_POST_DATA. + # + 'statistics_import' => { + create_sql => ' + CREATE TABLE dump_test.has_stats + AS SELECT g.g AS x, g.g / 2 AS y FROM generate_series(1,100) AS g(g); + CREATE MATERIALIZED VIEW dump_test.has_stats_mv AS SELECT * FROM dump_test.has_stats; + CREATE INDEX dup_test_post_data_ix ON dump_test.has_stats((x - 1)); + ANALYZE dump_test.has_stats, dump_test.has_stats_mv;', + regexp => qr/pg_catalog.pg_restore_attribute_stats/, + like => { + %full_runs, + %dump_test_schema_runs, + no_data_no_schema => 1, + no_schema => 1, + section_data => 1, + section_post_data => 1, + statistics_only => 1, + }, + unlike => { + exclude_dump_test_schema => 1, + no_statistics => 1, + only_dump_measurement => 1, + schema_only => 1, + }, + }, + + # + # While attribute stats (aka pg_statistic stats) only appear for tables + # that have been analyzed, all tables will have relation stats because + # those come from pg_class. + # + 'relstats_on_unanalyzed_tables' => { + regexp => qr/pg_catalog.pg_restore_relation_stats/, + + like => { + %full_runs, + %dump_test_schema_runs, + no_data_no_schema => 1, + no_schema => 1, + only_dump_test_table => 1, + role => 1, + role_parallel => 1, + section_data => 1, + section_post_data => 1, + statistics_only => 1, + }, + unlike => { + no_statistics => 1, + schema_only => 1, + }, + }, + # CREATE TABLE with partitioned table and various AMs. One # partition uses the same default as the parent, and a second # uses its own AM. diff --git a/src/bin/pg_upgrade/dump.c b/src/bin/pg_upgrade/dump.c index 477436290e8..204cd0e5361 100644 --- a/src/bin/pg_upgrade/dump.c +++ b/src/bin/pg_upgrade/dump.c @@ -56,10 +56,11 @@ generate_old_dump(void) snprintf(log_file_name, sizeof(log_file_name), DB_DUMP_LOG_FILE_MASK, old_db->db_oid); parallel_exec_prog(log_file_name, NULL, - "\"%s/pg_dump\" %s --schema-only --quote-all-identifiers " + "\"%s/pg_dump\" %s --no-data %s --quote-all-identifiers " "--binary-upgrade --format=custom %s --no-sync --file=\"%s/%s\" %s", new_cluster.bindir, cluster_conn_opts(&old_cluster), log_opts.verbose ? "--verbose" : "", + user_opts.do_statistics ? "" : "--no-statistics", log_opts.dumpdir, sql_file_name, escaped_connstr.data); diff --git a/src/bin/pg_upgrade/option.c b/src/bin/pg_upgrade/option.c index 548ea4e6236..2d560cf7183 100644 --- a/src/bin/pg_upgrade/option.c +++ b/src/bin/pg_upgrade/option.c @@ -60,6 +60,7 @@ parseCommandLine(int argc, char *argv[]) {"copy", no_argument, NULL, 2}, {"copy-file-range", no_argument, NULL, 3}, {"sync-method", required_argument, NULL, 4}, + {"no-statistics", no_argument, NULL, 5}, {NULL, 0, NULL, 0} }; @@ -70,6 +71,7 @@ parseCommandLine(int argc, char *argv[]) user_opts.do_sync = true; user_opts.transfer_mode = TRANSFER_MODE_COPY; + user_opts.do_statistics = true; os_info.progname = get_progname(argv[0]); @@ -212,6 +214,10 @@ parseCommandLine(int argc, char *argv[]) user_opts.sync_method = pg_strdup(optarg); break; + case 5: + user_opts.do_statistics = false; + break; + default: fprintf(stderr, _("Try \"%s --help\" for more information.\n"), os_info.progname); @@ -306,6 +312,7 @@ usage(void) printf(_(" --clone clone instead of copying files to new cluster\n")); printf(_(" --copy copy files to new cluster (default)\n")); printf(_(" --copy-file-range copy files to new cluster with copy_file_range\n")); + printf(_(" --no-statistics do not import statistics from old cluster\n")); printf(_(" --sync-method=METHOD set method for syncing files to disk\n")); printf(_(" -?, --help show this help, then exit\n")); printf(_("\n" diff --git a/src/bin/pg_upgrade/pg_upgrade.h b/src/bin/pg_upgrade/pg_upgrade.h index 5ad693eaa70..a2333eb6dc3 100644 --- a/src/bin/pg_upgrade/pg_upgrade.h +++ b/src/bin/pg_upgrade/pg_upgrade.h @@ -327,6 +327,7 @@ typedef struct int jobs; /* number of processes/threads to use */ char *socketdir; /* directory to use for Unix sockets */ char *sync_method; + bool do_statistics; /* carry over statistics from old cluster */ } UserOpts; typedef struct diff --git a/src/bin/pg_upgrade/t/002_pg_upgrade.pl b/src/bin/pg_upgrade/t/002_pg_upgrade.pl index 649be522dc0..583479f4ca2 100644 --- a/src/bin/pg_upgrade/t/002_pg_upgrade.pl +++ b/src/bin/pg_upgrade/t/002_pg_upgrade.pl @@ -275,6 +275,9 @@ sub filter_dump $node_params{extra} = \@initdb_params; $newnode->init(%node_params); +# Stabilize stats for comparison. +$newnode->append_conf('postgresql.conf', 'autovacuum = off'); + my $newbindir = $newnode->config_data('--bindir'); my $oldbindir = $oldnode->config_data('--bindir'); @@ -311,6 +314,10 @@ sub filter_dump } } +# Stabilize stats before pg_dumpall. +$oldnode->append_conf('postgresql.conf', 'autovacuum = off'); +$oldnode->restart; + # Take a dump before performing the upgrade as a base comparison. Note # that we need to use pg_dumpall from the new node here. my @dump_command = ( diff --git a/src/test/perl/PostgreSQL/Test/AdjustUpgrade.pm b/src/test/perl/PostgreSQL/Test/AdjustUpgrade.pm index 90c74bde40b..63dd877e130 100644 --- a/src/test/perl/PostgreSQL/Test/AdjustUpgrade.pm +++ b/src/test/perl/PostgreSQL/Test/AdjustUpgrade.pm @@ -294,6 +294,11 @@ sub adjust_old_dumpfile # Version comments will certainly not match. $dump =~ s/^-- Dumped from database version.*\n//mg; + # Same with version argument to pg_restore_relation_stats() or + # pg_restore_attribute_stats(). + $dump =~ s ['version', '${old_version}\d{4}'::integer,] + ['version', '000000'::integer,]mg; + if ($old_version < 16) { # Fix up some view queries that no longer require table-qualification. @@ -633,6 +638,10 @@ sub adjust_new_dumpfile { $dump =~ s/XMLSERIALIZE\((.*)? NO INDENT\)/XMLSERIALIZE\($1\)/mg; } + # Same with version argument to pg_restore_relation_stats() or + # pg_restore_attribute_stats(). + $dump =~ s ['version', '\d{6}'::integer,] + ['version', '000000'::integer,]mg; if ($old_version < 14) { diff --git a/src/test/recovery/t/027_stream_regress.pl b/src/test/recovery/t/027_stream_regress.pl index 7bd9054a618..8a63734be06 100644 --- a/src/test/recovery/t/027_stream_regress.pl +++ b/src/test/recovery/t/027_stream_regress.pl @@ -107,7 +107,7 @@ [ 'pg_dumpall', '-f', $outputdir . '/primary.dump', '--restrict-key=test', - '--no-sync', '-p', $node_primary->port, + '--no-sync', '--no-statistics', '-p', $node_primary->port, '--no-unlogged-table-data' # if unlogged, standby has schema only ], 'dump primary server'); @@ -115,7 +115,7 @@ [ 'pg_dumpall', '-f', $outputdir . '/standby.dump', '--restrict-key=test', - '--no-sync', '-p', $node_standby_1->port + '--no-sync', '--no-statistics', '-p', $node_standby_1->port ], 'dump standby server'); command_ok( diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 84d88f895d6..9084972bf52 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2378,6 +2378,7 @@ RelMapFile RelMapping RelOptInfo RelOptKind +RelStatsInfo RelToCheck RelToCluster RelabelType From d9bf1ab78e580d0cf51b46c54d7e26d3bb6c3fc2 Mon Sep 17 00:00:00 2001 From: reshke kirill Date: Mon, 26 Feb 2024 20:10:21 +0000 Subject: [PATCH 13/54] Allow non-superuser to cancel superuser tasks. Patch adds new pre-defined role `pg_signal_autovacuum` and tap tests. Role, granted with `pg_signal_autovacuum` is able to cancel running autovacuum worker. Note that to cancel all other type of queries role need to be granted with `pg_singal_backend`. --- src/backend/storage/ipc/signalfuncs.c | 28 ++++++--- src/include/catalog/pg_authid.dat | 5 ++ src/test/signals/.gitignore | 2 + src/test/signals/Makefile | 23 ++++++++ src/test/signals/t/001_signal_autovacuum.pl | 63 +++++++++++++++++++++ 5 files changed, 113 insertions(+), 8 deletions(-) create mode 100644 src/test/signals/.gitignore create mode 100644 src/test/signals/Makefile create mode 100644 src/test/signals/t/001_signal_autovacuum.pl diff --git a/src/backend/storage/ipc/signalfuncs.c b/src/backend/storage/ipc/signalfuncs.c index 88e9bf8125d..d398961be1f 100644 --- a/src/backend/storage/ipc/signalfuncs.c +++ b/src/backend/storage/ipc/signalfuncs.c @@ -78,15 +78,27 @@ pg_signal_backend(int pid, int sig) * Only allow superusers to signal superuser-owned backends. Any process * not advertising a role might have the importance of a superuser-owned * backend, so treat it that way. + * In later case, check if pid is actually autovacuum_worker, and allow + * to signal if role has proper priviledge. */ - if ((!OidIsValid(proc->roleId) || superuser_arg(proc->roleId)) && - !superuser()) - return SIGNAL_BACKEND_NOSUPERUSER; - - /* Users can signal backends they have role membership in. */ - if (!has_privs_of_role(GetUserId(), proc->roleId) && - !has_privs_of_role(GetUserId(), ROLE_PG_SIGNAL_BACKEND)) - return SIGNAL_BACKEND_NOPERMISSION; + if (!superuser()) { + if (!OidIsValid(proc->roleId)) { + LocalPgBackendStatus *local_beentry; + local_beentry = pgstat_get_local_beentry_by_backend_id(proc->backendId); + + if (!(local_beentry && local_beentry->backendStatus.st_backendType == B_AUTOVAC_WORKER && + has_privs_of_role(GetUserId(), ROLE_PG_SIGNAL_AUTOVACUUM))) + return SIGNAL_BACKEND_NOSUPERUSER; + } else { + if (superuser_arg(proc->roleId)) + return SIGNAL_BACKEND_NOSUPERUSER; + + /* Users can signal backends they have role membership in. */ + if (!has_privs_of_role(GetUserId(), proc->roleId) && + !has_privs_of_role(GetUserId(), ROLE_PG_SIGNAL_BACKEND)) + return SIGNAL_BACKEND_NOPERMISSION; + } + } /* * Can the process we just validated above end, followed by the pid being diff --git a/src/include/catalog/pg_authid.dat b/src/include/catalog/pg_authid.dat index bf00815c14e..63718d6a892 100644 --- a/src/include/catalog/pg_authid.dat +++ b/src/include/catalog/pg_authid.dat @@ -99,5 +99,10 @@ rolcreaterole => 'f', rolcreatedb => 'f', rolcanlogin => 'f', rolreplication => 'f', rolbypassrls => 'f', rolconnlimit => '-1', rolpassword => '_null_', rolvaliduntil => '_null_' }, +{ oid => '6312', oid_symbol => 'ROLE_PG_SIGNAL_AUTOVACUUM', + rolname => 'pg_signal_autovacuum', rolsuper => 'f', rolinherit => 't', + rolcreaterole => 'f', rolcreatedb => 'f', rolcanlogin => 'f', + rolreplication => 'f', rolbypassrls => 'f', rolconnlimit => '-1', + rolpassword => '_null_', rolvaliduntil => '_null_' }, ] diff --git a/src/test/signals/.gitignore b/src/test/signals/.gitignore new file mode 100644 index 00000000000..871e943d50e --- /dev/null +++ b/src/test/signals/.gitignore @@ -0,0 +1,2 @@ +# Generated by test suite +/tmp_check/ diff --git a/src/test/signals/Makefile b/src/test/signals/Makefile new file mode 100644 index 00000000000..688f6a7e9e0 --- /dev/null +++ b/src/test/signals/Makefile @@ -0,0 +1,23 @@ +#------------------------------------------------------------------------- +# +# Makefile for src/test/recovery +# +# Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group +# Portions Copyright (c) 1994, Regents of the University of California +# +# src/test/recovery/Makefile +# +#------------------------------------------------------------------------- + +subdir = src/test/signals +top_builddir = ../../.. +include $(top_builddir)/src/Makefile.global + +check: + $(prove_check) + +installcheck: + $(prove_installcheck) + +clean distclean maintainer-clean: + rm -rf tmp_check diff --git a/src/test/signals/t/001_signal_autovacuum.pl b/src/test/signals/t/001_signal_autovacuum.pl new file mode 100644 index 00000000000..ec4b3e78de6 --- /dev/null +++ b/src/test/signals/t/001_signal_autovacuum.pl @@ -0,0 +1,63 @@ +# Copyright (c) 2024-2024, PostgreSQL Global Development Group + +# Minimal test testing pg_signal_autovacuum role. +use strict; +use warnings; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +# Initialize primary node +my $node_primary = PostgreSQL::Test::Cluster->new('primary'); +$node_primary->init(); +$node_primary->start; + +$node_primary->safe_psql('postgres', + " + CREATE DATABASE regress; + CREATE ROLE psa_reg_role_1; + CREATE ROLE psa_reg_role_2; + GRANT pg_signal_backend TO psa_reg_role_1; + GRANT pg_signal_autovacuum TO psa_reg_role_2; +"); + +# Create some content on primary and set autovacuum setting such that +# it would be triggered. +$node_primary->safe_psql('regress', + " + CREATE TABLE tab_int(i int); + INSERT INTO tab_int SELECT * FROM generate_series(1, 1000000); + ALTER SYSTEM SET autovacuum_vacuum_cost_limit TO 1; + ALTER SYSTEM SET autovacuum_vacuum_cost_delay TO 100; + ALTER SYSTEM SET autovacuum_naptime TO 1; +"); + +$node_primary->restart; + +#wait for autovac to start. + +sleep 1; + +my $res_pid = $node_primary->safe_psql('regress', + " + SELECT pid FROM pg_stat_activity WHERE backend_type = 'autovacuum worker' and datname = 'regress';; +"); + + +my ($res_reg_psa_1, $stdout_reg_psa_1, $stderr_reg_psa_1) = $node_primary->psql('regress', + " + SET ROLE psa_reg_role_1; + SELECT pg_terminate_backend($res_pid); +"); + +ok($res_reg_psa_1 != 0, "should fail for non pg_signal_autovacuum"); +like($stderr_reg_psa_1, qr/Only roles with the SUPERUSER attribute may terminate processes of roles with the SUPERUSER attribute./, "matches"); + +my ($res_reg_psa_2, $stdout_reg_psa_2, $stderr_reg_psa_2) = $node_primary->psql('regress', " + SET ROLE psa_reg_role_2; + SELECT pg_terminate_backend($res_pid); +"); + +ok($res_reg_psa_2 == 0, "should success for pg_signal_autovacuum"); + +done_testing(); From df290882136a7ab552e217827afd51e7aa0693ff Mon Sep 17 00:00:00 2001 From: reshke kirill Date: Mon, 26 Feb 2024 21:21:31 +0000 Subject: [PATCH 14/54] Add debain for PostgreSQL 17 Bump deb to bet3 --- debian/changelog | 35 ++ debian/clean | 3 + debian/control | 299 ++++++++++++++++++ debian/copyright | 270 ++++++++++++++++ debian/libecpg-compat3.install | 1 + debian/libecpg-compat3.symbols | 44 +++ debian/libecpg-dev.install | 18 ++ debian/libecpg6.install | 2 + debian/libecpg6.symbols | 31 ++ debian/libpgtypes3.install | 1 + debian/libpgtypes3.symbols | 48 +++ debian/libpq-dev.dirs | 1 + debian/libpq-dev.install | 11 + debian/libpq5.install | 2 + debian/libpq5.symbols | 207 ++++++++++++ debian/patches/50-per-version-dirs.patch | 29 ++ .../patches/51-default-sockets-in-var.patch | 20 ++ debian/patches/52-tutorial-README.patch | 16 + .../53-pg_service.conf_directory_doc.patch | 19 ++ ...bian-alternatives-for-external-tools.patch | 28 ++ debian/patches/70-history | 13 + debian/patches/autoconf2.69 | 7 + debian/patches/extension_destdir | 228 +++++++++++++ debian/patches/filter-debug-prefix-map | 34 ++ debian/patches/focal-arm64-outline-atomics | 23 ++ debian/patches/hurd-iovec | 26 ++ debian/patches/jit-s390x | 96 ++++++ debian/patches/libpgport-pkglibdir | 82 +++++ .../pgstat-report-conflicts-immediately.patch | 37 +++ debian/patches/series | 14 + debian/po/POTFILES.in | 1 + debian/po/de.po | 37 +++ debian/po/es.po | 58 ++++ debian/po/fr.po | 39 +++ debian/po/it.po | 37 +++ debian/po/nl.po | 40 +++ debian/po/pt.po | 39 +++ debian/po/pt_BR.po | 37 +++ debian/po/ro.po | 50 +++ debian/po/ru.po | 39 +++ debian/po/sv.po | 36 +++ debian/postgresql-17.install | 62 ++++ debian/postgresql-17.lintian-overrides | 20 ++ debian/postgresql-17.postinst | 13 + debian/postgresql-17.postrm | 80 +++++ debian/postgresql-17.preinst | 18 ++ debian/postgresql-17.prerm | 16 + debian/postgresql-17.templates | 7 + debian/postgresql-client-17.install | 48 +++ debian/postgresql-client-17.lintian-overrides | 3 + debian/postgresql-client-17.postinst | 13 + debian/postgresql-client-17.prerm | 12 + debian/postgresql-doc-17.doc-base | 18 ++ debian/postgresql-doc-17.install | 2 + debian/postgresql-doc-17.postinst | 30 ++ debian/postgresql-doc-17.prerm | 18 ++ debian/postgresql-plperl-17.install | 6 + debian/postgresql-plperl-17.lintian-overrides | 1 + debian/postgresql-plpython3-17.install | 6 + .../postgresql-plpython3-17.lintian-overrides | 1 + debian/postgresql-pltcl-17.install | 3 + debian/postgresql-pltcl-17.lintian-overrides | 1 + debian/postgresql-server-dev-17.install | 5 + ...postgresql-server-dev-17.lintian-overrides | 1 + debian/rules | 5 + debian/source/format | 1 + debian/source/lintian-overrides | 4 + debian/tests/Makefile.regress | 5 + debian/tests/control | 21 ++ debian/tests/installcheck | 41 +++ debian/tests/run-testsuite | 5 + debian/watch | 4 + 72 files changed, 2528 insertions(+) create mode 100644 debian/changelog create mode 100644 debian/clean create mode 100644 debian/control create mode 100644 debian/copyright create mode 100644 debian/libecpg-compat3.install create mode 100644 debian/libecpg-compat3.symbols create mode 100644 debian/libecpg-dev.install create mode 100644 debian/libecpg6.install create mode 100644 debian/libecpg6.symbols create mode 100644 debian/libpgtypes3.install create mode 100644 debian/libpgtypes3.symbols create mode 100644 debian/libpq-dev.dirs create mode 100644 debian/libpq-dev.install create mode 100644 debian/libpq5.install create mode 100644 debian/libpq5.symbols create mode 100644 debian/patches/50-per-version-dirs.patch create mode 100644 debian/patches/51-default-sockets-in-var.patch create mode 100644 debian/patches/52-tutorial-README.patch create mode 100644 debian/patches/53-pg_service.conf_directory_doc.patch create mode 100644 debian/patches/54-debian-alternatives-for-external-tools.patch create mode 100644 debian/patches/70-history create mode 100644 debian/patches/autoconf2.69 create mode 100644 debian/patches/extension_destdir create mode 100644 debian/patches/filter-debug-prefix-map create mode 100644 debian/patches/focal-arm64-outline-atomics create mode 100644 debian/patches/hurd-iovec create mode 100644 debian/patches/jit-s390x create mode 100644 debian/patches/libpgport-pkglibdir create mode 100644 debian/patches/pgstat-report-conflicts-immediately.patch create mode 100644 debian/patches/series create mode 100644 debian/po/POTFILES.in create mode 100644 debian/po/de.po create mode 100644 debian/po/es.po create mode 100644 debian/po/fr.po create mode 100644 debian/po/it.po create mode 100644 debian/po/nl.po create mode 100644 debian/po/pt.po create mode 100644 debian/po/pt_BR.po create mode 100644 debian/po/ro.po create mode 100644 debian/po/ru.po create mode 100644 debian/po/sv.po create mode 100755 debian/postgresql-17.install create mode 100644 debian/postgresql-17.lintian-overrides create mode 100644 debian/postgresql-17.postinst create mode 100644 debian/postgresql-17.postrm create mode 100644 debian/postgresql-17.preinst create mode 100644 debian/postgresql-17.prerm create mode 100644 debian/postgresql-17.templates create mode 100644 debian/postgresql-client-17.install create mode 100644 debian/postgresql-client-17.lintian-overrides create mode 100644 debian/postgresql-client-17.postinst create mode 100644 debian/postgresql-client-17.prerm create mode 100644 debian/postgresql-doc-17.doc-base create mode 100644 debian/postgresql-doc-17.install create mode 100644 debian/postgresql-doc-17.postinst create mode 100644 debian/postgresql-doc-17.prerm create mode 100755 debian/postgresql-plperl-17.install create mode 120000 debian/postgresql-plperl-17.lintian-overrides create mode 100755 debian/postgresql-plpython3-17.install create mode 120000 debian/postgresql-plpython3-17.lintian-overrides create mode 100644 debian/postgresql-pltcl-17.install create mode 120000 debian/postgresql-pltcl-17.lintian-overrides create mode 100644 debian/postgresql-server-dev-17.install create mode 120000 debian/postgresql-server-dev-17.lintian-overrides create mode 100755 debian/rules create mode 100644 debian/source/format create mode 100644 debian/source/lintian-overrides create mode 100644 debian/tests/Makefile.regress create mode 100644 debian/tests/control create mode 100755 debian/tests/installcheck create mode 100755 debian/tests/run-testsuite create mode 100644 debian/watch diff --git a/debian/changelog b/debian/changelog new file mode 100644 index 00000000000..e725b3f7fde --- /dev/null +++ b/debian/changelog @@ -0,0 +1,35 @@ +postgresql-17 (17~beta3-1.pgdg22.04+~20240822.0936.g6b1f78d) jammy-pgdg; urgency=medium + + * Rebuild for jammy-pgdg. + * Changes applied by generate-pgdg-source: + + Merge Debian packaging repository (17) with upstream (REL_17_STABLE). + + Moving binary packages to component 17. + + Use clang/llvm 15 for jit support. + + Remove tzdata-legacy from build and test dependencies. + + -- PostgreSQL on Debian and Ubuntu Wed, 07 Aug 2024 16:16:02 +0200 + +postgresql-17 (17~beta3-1) experimental; urgency=medium + + * Third beta version. + + -- Christoph Berg Wed, 07 Aug 2024 16:16:02 +0200 + +postgresql-17 (17~beta2-1) experimental; urgency=medium + + * Restrict systemtap-sdt-dev B-D to linux-any. + * Add libpq5 symbol PQgetCurrentTimeUSec. + + -- Christoph Berg Tue, 25 Jun 2024 14:03:14 +0200 + +postgresql-17 (17~beta1-1) experimental; urgency=medium + + * First beta version. + + -- Christoph Berg Wed, 22 May 2024 18:54:56 +0200 + +postgresql-17 (17~~devel20240509-1) experimental; urgency=medium + + * New major upstream version 17; packaging based on postgresql-16. + + -- Christoph Berg Thu, 09 May 2024 18:45:32 +0200 diff --git a/debian/clean b/debian/clean new file mode 100644 index 00000000000..a1e3d37a58a --- /dev/null +++ b/debian/clean @@ -0,0 +1,3 @@ +# regression tests debris +src/test/regress/regress.o +src/test/regress/regress.so diff --git a/debian/control b/debian/control new file mode 100644 index 00000000000..dace4c44839 --- /dev/null +++ b/debian/control @@ -0,0 +1,299 @@ +Source: postgresql-17 +Section: 17/database +Priority: optional +Maintainer: Debian PostgreSQL Maintainers +Uploaders: + Martin Pitt , + Peter Eisentraut , + Christoph Berg , +Standards-Version: 4.5.0 +Rules-Requires-Root: no +Build-Depends: + autoconf, + bison, + clang-15 [!alpha !hppa !hurd-amd64 !hurd-i386 !ia64 !kfreebsd-amd64 !kfreebsd-i386 !loong64 !m68k !powerpc !riscv64 !sh4 !sparc64 !x32] , + debhelper-compat (= 13), + dh-exec (>= 0.13~), + docbook-xml, + docbook-xsl (>= 1.77), + dpkg-dev (>= 1.16.1~), + flex, + gdb , + gettext, + libicu-dev, + libio-pty-perl , + libipc-run-perl , + libkrb5-dev, + libldap2-dev, + liblz4-dev, + libpam0g-dev | libpam-dev, + libperl-dev, + libreadline-dev, + libselinux1-dev [linux-any], + libssl-dev, + libsystemd-dev [linux-any], + libxml2-dev, + libxml2-utils, + libxslt1-dev, + libzstd-dev (>= 1.4.0) , + llvm-15-dev [!alpha !hppa !hurd-amd64 !hurd-i386 !ia64 !kfreebsd-amd64 !kfreebsd-i386 !loong64 !m68k !powerpc !riscv64 !sh4 !sparc64 !x32] , + lz4 | liblz4-tool, + mawk, + perl (>= 5.8), + pkgconf, + postgresql-common (>= 256~), + python3-dev, + systemtap-sdt-dev [linux-any], + tcl-dev, + tzdata , + uuid-dev, + xsltproc, + zlib1g-dev | libz-dev, + zstd (>= 1.4.0) , +Homepage: http://www.postgresql.org/ +Vcs-Browser: https://salsa.debian.org/postgresql/postgresql +Vcs-Git: https://salsa.debian.org/postgresql/postgresql.git -b 17 + +Package: libpq-dev +Build-Profiles: +Architecture: any +Section: 17/libdevel +Depends: + libpq5 (= ${binary:Version}), + libssl-dev, + ${misc:Depends}, + ${shlibs:Depends}, +Suggests: + postgresql-doc-17, +Description: header files for libpq5 (PostgreSQL library) + Header files and static library for compiling C programs to link + with the libpq library in order to communicate with a PostgreSQL + database backend. + . + PostgreSQL is an object-relational SQL database management system. + +Package: libpq5 +Build-Profiles: +Architecture: any +Section: 17/libs +Depends: + ${misc:Depends}, + ${shlibs:Depends}, +Pre-Depends: + ${misc:Pre-Depends}, +Multi-Arch: same +Description: PostgreSQL C client library + libpq is a C library that enables user programs to communicate with + the PostgreSQL database server. The server can be on another machine + and accessed through TCP/IP. This version of libpq is compatible + with servers from PostgreSQL 8.2 or later. + . + This package contains the run-time library, needed by packages using + libpq. + . + PostgreSQL is an object-relational SQL database management system. + +Package: libecpg6 +Build-Profiles: +Architecture: any +Section: 17/libs +Depends: + ${misc:Depends}, + ${shlibs:Depends}, +Pre-Depends: + ${misc:Pre-Depends}, +Multi-Arch: same +Description: run-time library for ECPG programs + The libecpg shared library is used by programs built with ECPG + (Embedded PostgreSQL for C). + . + PostgreSQL is an object-relational SQL database management system. + +Package: libecpg-dev +Build-Profiles: +Architecture: any +Section: 17/libdevel +Depends: + libecpg-compat3 (= ${binary:Version}), + libecpg6 (= ${binary:Version}), + libpgtypes3 (= ${binary:Version}), + libpq-dev, + ${misc:Depends}, + ${shlibs:Depends}, +Description: development files for ECPG (Embedded PostgreSQL for C) + This package contains the necessary files to build ECPG (Embedded + PostgreSQL for C) programs. It includes the development libraries + and the preprocessor program ecpg. + . + PostgreSQL is an object-relational SQL database management system. + . + Install this package if you want to write C programs with SQL statements + embedded in them (rather than run by an external process). + +Package: libecpg-compat3 +Build-Profiles: +Architecture: any +Section: 17/libs +Depends: + ${misc:Depends}, + ${shlibs:Depends}, +Pre-Depends: + ${misc:Pre-Depends}, +Multi-Arch: same +Description: older version of run-time library for ECPG programs + The libecpg_compat shared library is used by programs built with ecpg. + (Embedded PostgreSQL for C). + . + PostgreSQL is an object-relational SQL database management system. + +Package: libpgtypes3 +Build-Profiles: +Architecture: any +Section: 17/libs +Depends: + ${misc:Depends}, + ${shlibs:Depends}, +Pre-Depends: + ${misc:Pre-Depends}, +Multi-Arch: same +Description: shared library libpgtypes for PostgreSQL 17 + The libpgtypes shared library is used by programs built with ecpg. + (Embedded PostgreSQL for C). + . + PostgreSQL is an object-relational SQL database management system. + +Package: postgresql-17 +Architecture: any +Depends: + locales | locales-all, + postgresql-client-17, + postgresql-common (>= 252~), + ssl-cert, + tzdata, + ${misc:Depends}, + ${shlibs:Depends}, +Provides: + postgresql-contrib-17, + postgresql-17-jit-llvm (= ${llvm:Version}) [!alpha !hppa !hurd-amd64 !hurd-i386 !ia64 !kfreebsd-amd64 !kfreebsd-i386 !loong64 !m68k !powerpc !riscv64 !sh4 !sparc64 !x32] , +Recommends: + sysstat, +Breaks: + dbconfig-common (<< 2.0.22~), +Description: The World's Most Advanced Open Source Relational Database + PostgreSQL, also known as Postgres, is a free and open-source relational + database management system (RDBMS) emphasizing extensibility and SQL + compliance. It features transactions with Atomicity, Consistency, Isolation, + Durability (ACID) properties, automatically updatable views, materialized + views, triggers, foreign keys, and stored procedures. It is designed to handle + a range of workloads, from single machines to data warehouses or Web services + with many concurrent users. + . + This package provides the database server for PostgreSQL 17.${cassert} +XB-Postgresql-Catversion: ${postgresql:Catversion} + +Package: postgresql-client-17 +Architecture: any +Multi-Arch: foreign +Depends: + libpq5 (>= ${source:Upstream-Version}), + postgresql-client-common (>= 182~), + sensible-utils, + ${misc:Depends}, + ${shlibs:Depends}, +Suggests: + postgresql-17, + postgresql-doc-17, +Provides: + postgresql-client, +Description: front-end programs for PostgreSQL 17 + This package contains client and administrative programs for + PostgreSQL: these are the interactive terminal client psql and + programs for creating and removing users and databases. + . + This is the client package for PostgreSQL 17. If you install + PostgreSQL 17 on a standalone machine, you need the server package + postgresql-17, too. On a network, you can install this package on + many client machines, while the server package may be installed on + only one machine. + . + PostgreSQL is an object-relational SQL database management system. + +Package: postgresql-server-dev-17 +Architecture: any +Section: 17/libdevel +Depends: + clang-${llvm:Version} [!alpha !hppa !hurd-amd64 !hurd-i386 !ia64 !kfreebsd-amd64 !kfreebsd-i386 !loong64 !m68k !powerpc !riscv64 !sh4 !sparc64 !x32] , + libpq-dev (>= 17~~), + llvm-${llvm:Version}-dev [!alpha !hppa !hurd-amd64 !hurd-i386 !ia64 !kfreebsd-amd64 !kfreebsd-i386 !loong64 !m68k !powerpc !riscv64 !sh4 !sparc64 !x32] , + postgresql-client-17, + postgresql-common (>= 142~), + ${misc:Depends}, + ${shlibs:Depends}, +Description: development files for PostgreSQL 17 server-side programming + Header files for compiling SSI code to link into PostgreSQL's backend; for + example, for C functions to be called from SQL. + . + This package also contains the Makefiles necessary for building add-on + modules of PostgreSQL, which would otherwise have to be built in the + PostgreSQL source-code tree. + . + PostgreSQL is an object-relational SQL database management system. + +Package: postgresql-doc-17 +Architecture: all +Build-Profiles: +Multi-Arch: foreign +Section: 17/doc +Depends: + ${misc:Depends}, +Description: documentation for the PostgreSQL database management system + This package contains all README files, user manual, and examples for + PostgreSQL 17. The manual is in HTML format. + . + PostgreSQL is an object-relational SQL database management system. + +Package: postgresql-plperl-17 +Architecture: any +Depends: + perl, + postgresql-17 (= ${binary:Version}), + ${misc:Depends}, + ${shlibs:Depends}, +Provides: + postgresql-plperl, +Description: PL/Perl procedural language for PostgreSQL 17 + PL/Perl enables an SQL developer to write procedural language functions + for PostgreSQL 17 in Perl. You need this package if you have any + PostgreSQL 17 functions that use the languages plperl or plperlu. + . + PostgreSQL is an object-relational SQL database management system. + +Package: postgresql-plpython3-17 +Architecture: any +Depends: + postgresql-17 (= ${binary:Version}), + ${misc:Depends}, + ${shlibs:Depends}, +Provides: + postgresql-plpython3, +Description: PL/Python 3 procedural language for PostgreSQL 17 + PL/Python 3 enables an SQL developer to write procedural language functions + for PostgreSQL 17 in Python 3. You need this package if you have any + PostgreSQL 17 functions that use the languages plpython3 or plpython3u. + . + PostgreSQL is an object-relational SQL database management system. + +Package: postgresql-pltcl-17 +Architecture: any +Depends: + postgresql-17 (= ${binary:Version}), + ${misc:Depends}, + ${shlibs:Depends}, +Provides: + postgresql-pltcl, +Description: PL/Tcl procedural language for PostgreSQL 17 + PL/Tcl enables an SQL developer to write procedural language functions + for PostgreSQL 17 in Tcl. You need this package if you have any + PostgreSQL 17 functions that use the languages pltcl or pltclu. + . + PostgreSQL is an object-relational SQL database management system. diff --git a/debian/copyright b/debian/copyright new file mode 100644 index 00000000000..df27d1ff61e --- /dev/null +++ b/debian/copyright @@ -0,0 +1,270 @@ +Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Upstream-Name: PostgreSQL +Source: https://www.postgresql.org/ftp/source/ + +### PostgreSQL copyrights ### + +Files: * +Copyright: Portions Copyright (c) 1996-2019, PostgreSQL Global Development Group + Portions Copyright (c) 1994, The Regents of the University of California +License: PostgreSQL + +Files: src/backend/regex/* +Copyright: + Copyright (c) 2013-2022, PostgreSQL Global Development Group + Copyright (c) 1998, 1999 Henry Spencer. All rights reserved. + Copyright (c) 1998 Sun Microsystems, Inc. + Copyright (c) 1998, 1999 by Scriptics Corporation. +License: PostgreSQL and Custom-regex and Tcl + +Files: src/bin/pg_dump/* +Copyright: Portions Copyright (c) 2000, Philip Warner +License: Custom-pg_dump + Rights are granted to use this software in any way so long as this notice is + not removed. The author is not responsible for loss or damages that may + result from its use. + +Files: src/backend/snowball/libstemmer/* src/include/snowball/libstemmer/* +Copyright: Copyright (c) 2001, Dr Martin Porter, + Copyright (c) 2002, Richard Boulton. +License: BSD-3-Clause +Comment: + This module uses the word stemming code developed by the Snowball project, + http://snowballstem.org which is released by them under a BSD-style license. + Postgres' files under src/backend/snowball/libstemmer/ and + src/include/snowball/libstemmer/ are taken directly from the Snowball files, + with only some minor adjustments of file inclusions. + See src/backend/snowball/README for details. + +Files: src/backend/utils/mb/Unicode/*.txt +Copyright: Copyright (C) 2001 earthian@tama.or.jp, All Rights Reserved. + Copyright (C) 2001 I'O, All Rights Reserved. + Copyright (C) 2006 Project X0213, All Rights Reserved. +License: Custom-Unicode + You can use, modify, distribute this table freely. + +Files: + src/common/sha2.c + src/include/common/sha2.h +Copyright: Copyright (c) 2000-2001, Aaron D. Gifford +License: BSD-3-Clause + +### contrib copyrights ### + +Files: contrib/fuzzystrmatch/dmetaphone.c +Copyright: Copyright 2000, Maurice Aubrey + Copyright 2003, North Carolina State Highway Patrol +License: double-metaphone + This module is free software; you may redistribute it and/or + modify it under the same terms as Perl itself. + . + All rights reserved. + . + Permission to use, copy, modify, and distribute this software and its + documentation for any purpose, without fee, and without a written agreement + is hereby granted, provided that the above copyright notice and this + paragraph and the following two paragraphs appear in all copies. + . + IN NO EVENT SHALL THE NORTH CAROLINA STATE HIGHWAY PATROL BE LIABLE TO ANY + PARTY FOR DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, + INCLUDING LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS + DOCUMENTATION, EVEN IF THE NORTH CAROLINA STATE HIGHWAY PATROL HAS BEEN + ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + . + THE NORTH CAROLINA STATE HIGHWAY PATROL SPECIFICALLY DISCLAIMS ANY + WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED + HEREUNDER IS ON AN "AS IS" BASIS, AND THE NORTH CAROLINA STATE HIGHWAY PATROL + HAS NO OBLIGATIONS TO PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR + MODIFICATIONS. + . + The license of Perl is: + . + This program is free software; you can redistribute it and/or modify + it under the terms of either: + . + a) the GNU General Public License as published by the Free Software + Foundation; either version 1, or (at your option) any later + version, or + . + b) the "Artistic License" which comes with Perl. + . + On Debian GNU/Linux systems, the complete text of the GNU General + Public License version 1 can be found in + `/usr/share/common-licenses/GPL-1' and the Artistic Licence in + `/usr/share/common-licenses/Artistic'. + +Files: + contrib/pageinspect/btreefuncs.c + contrib/pgrowlocks/* + contrib/pgstattuple/* +Copyright: Copyright (c) 2006 Satoshi Nagayasu + Copyright (c) 2001, 2002, 2005-2006 Tatsuo Ishii +License: nagaysau-ishii + Permission to use, copy, modify, and distribute this software and + its documentation for any purpose, without fee, and without a + written agreement is hereby granted, provided that the above + copyright notice and this paragraph and the following two + paragraphs appear in all copies. + . + IN NO EVENT SHALL THE AUTHOR BE LIABLE TO ANY PARTY FOR DIRECT, + INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS + DOCUMENTATION, EVEN IF THE UNIVERSITY OF CALIFORNIA HAS BEEN ADVISED + OF THE POSSIBILITY OF SUCH DAMAGE. + . + THE AUTHOR SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON AN "AS + IS" BASIS, AND THE AUTHOR HAS NO OBLIGATIONS TO PROVIDE MAINTENANCE, + SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + +Files: + contrib/pgcrypto/crypt-des.c +Copyright: Copyright (c) 1994 David Burren + Copyright (C) 1995, 1996, 1997, and 1998 WIDE Project. +License: BSD-3-clause + +Files: + contrib/pgcrypto/mbuf.* + contrib/pgcrypto/openssl.c + contrib/pgcrypto/pgcrypto.* + contrib/pgcrypto/pgp* + contrib/pgcrypto/px* +Copyright: Copyright (c) 2001, 2005 Marko Kreen +License: BSD-2-clause + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + . + THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + SUCH DAMAGE. + +### licenses ### + +License: PostgreSQL + Permission to use, copy, modify, and distribute this software and its + documentation for any purpose, without fee, and without a written agreement + is hereby granted, provided that the above copyright notice and this + paragraph and the following two paragraphs appear in all copies. + . + IN NO EVENT SHALL THE UNIVERSITY OF CALIFORNIA BE LIABLE TO ANY PARTY FOR + DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS + DOCUMENTATION, EVEN IF THE UNIVERSITY OF CALIFORNIA HAS BEEN ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + . + THE UNIVERSITY OF CALIFORNIA SPECIFICALLY DISCLAIMS ANY WARRANTIES, + INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS + ON AN "AS IS" BASIS, AND THE UNIVERSITY OF CALIFORNIA HAS NO OBLIGATIONS TO + PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + +License: BSD-3-Clause + All rights reserved. + . + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + . + 1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + . + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + . + 3. Neither the name of the copyright holder nor the names of its contributors + may be used to endorse or promote products derived from this software without + specific prior written permission. + . + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +License: Custom-regex + Development of this software was funded, in part, by Cray Research Inc., + UUNET Communications Services Inc., Sun Microsystems Inc., and Scriptics + Corporation, none of whom are responsible for the results. The author + thanks all of them. + . + Redistribution and use in source and binary forms -- with or without + modification -- are permitted for any purpose, provided that + redistributions in source form retain this entire copyright notice and + indicate the origin and nature of any modifications. + . + I'd appreciate being given credit for this package in the documentation + of software which uses it, but that is not a requirement. + . + THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, + INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL + HENRY SPENCER BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +License: Tcl + This software is copyrighted by the Regents of the University of + California, Sun Microsystems, Inc., Scriptics Corporation, ActiveState + Corporation and other parties. The following terms apply to all files + associated with the software unless explicitly disclaimed in + individual files. + . + The authors hereby grant permission to use, copy, modify, distribute, + and license this software and its documentation for any purpose, provided + that existing copyright notices are retained in all copies and that this + notice is included verbatim in any distributions. No written agreement, + license, or royalty fee is required for any of the authorized uses. + Modifications to this software may be copyrighted by their authors + and need not follow the licensing terms described here, provided that + the new terms are clearly indicated on the first page of each file where + they apply. + . + IN NO EVENT SHALL THE AUTHORS OR DISTRIBUTORS BE LIABLE TO ANY PARTY + FOR DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES + ARISING OUT OF THE USE OF THIS SOFTWARE, ITS DOCUMENTATION, OR ANY + DERIVATIVES THEREOF, EVEN IF THE AUTHORS HAVE BEEN ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + . + THE AUTHORS AND DISTRIBUTORS SPECIFICALLY DISCLAIM ANY WARRANTIES, + INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE, AND NON-INFRINGEMENT. THIS SOFTWARE + IS PROVIDED ON AN "AS IS" BASIS, AND THE AUTHORS AND DISTRIBUTORS HAVE + NO OBLIGATION TO PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR + MODIFICATIONS. + . + GOVERNMENT USE: If you are acquiring this software on behalf of the + U.S. government, the Government shall have only "Restricted Rights" + in the software and related documentation as defined in the Federal + Acquisition Regulations (FARs) in Clause 52.227.19 (c) (2). If you + are acquiring the software on behalf of the Department of Defense, the + software shall be classified as "Commercial Computer Software" and the + Government shall have only "Restricted Rights" as defined in Clause + 252.227-7013 (c) (1) of DFARs. Notwithstanding the foregoing, the + authors grant the U.S. Government and others acting in its behalf + permission to use and distribute the software in accordance with the + terms specified in this license. diff --git a/debian/libecpg-compat3.install b/debian/libecpg-compat3.install new file mode 100644 index 00000000000..c69d50d8467 --- /dev/null +++ b/debian/libecpg-compat3.install @@ -0,0 +1 @@ +usr/lib/*/libecpg_compat.so.3* diff --git a/debian/libecpg-compat3.symbols b/debian/libecpg-compat3.symbols new file mode 100644 index 00000000000..3bd41e03e15 --- /dev/null +++ b/debian/libecpg-compat3.symbols @@ -0,0 +1,44 @@ +libecpg_compat.so.3 libecpg-compat3 #MINVER# +* Build-Depends-Package: libecpg-dev + ECPG_informix_get_var@Base 0 + ECPG_informix_reset_sqlca@Base 9.0~ + ECPG_informix_set_var@Base 0 + byleng@Base 0 + decadd@Base 0 + deccmp@Base 0 + deccopy@Base 0 + deccvasc@Base 0 + deccvdbl@Base 0 + deccvint@Base 0 + deccvlong@Base 0 + decdiv@Base 0 + decmul@Base 0 + decsub@Base 0 + dectoasc@Base 0 + dectodbl@Base 0 + dectoint@Base 0 + dectolong@Base 0 + dtcurrent@Base 0 + dtcvasc@Base 0 + dtcvfmtasc@Base 0 + dtsub@Base 0 + dttoasc@Base 0 + dttofmtasc@Base 0 + intoasc@Base 0 + ldchar@Base 0 + rdatestr@Base 0 + rdayofweek@Base 0 + rdefmtdate@Base 0 + rfmtdate@Base 0 + rfmtlong@Base 0 + rgetmsg@Base 0 + risnull@Base 0 + rjulmdy@Base 0 + rmdyjul@Base 0 + rsetnull@Base 0 + rstrdate@Base 0 + rtoday@Base 0 + rtypalign@Base 0 + rtypmsize@Base 0 + rtypwidth@Base 0 + rupshift@Base 0 diff --git a/debian/libecpg-dev.install b/debian/libecpg-dev.install new file mode 100644 index 00000000000..1935a0d885a --- /dev/null +++ b/debian/libecpg-dev.install @@ -0,0 +1,18 @@ +usr/include/postgresql/ecpg*.h +usr/include/postgresql/informix/* +usr/include/postgresql/pgtypes*.h +usr/include/postgresql/sql3types.h +usr/include/postgresql/sqlca.h +usr/include/postgresql/sqlda*.h +usr/lib/*/libecpg.a +usr/lib/*/libecpg.so +usr/lib/*/libecpg_compat.a +usr/lib/*/libecpg_compat.so +usr/lib/*/libpgtypes.a +usr/lib/*/libpgtypes.so +usr/lib/*/pkgconfig/libecpg.pc +usr/lib/*/pkgconfig/libecpg_compat.pc +usr/lib/*/pkgconfig/libpgtypes.pc +usr/lib/postgresql/*/bin/ecpg usr/bin +usr/share/locale/*/*/ecpg-*.mo +usr/share/postgresql/*/man/man1/ecpg.1* /usr/share/man/man1 diff --git a/debian/libecpg6.install b/debian/libecpg6.install new file mode 100644 index 00000000000..8ec1cee0f03 --- /dev/null +++ b/debian/libecpg6.install @@ -0,0 +1,2 @@ +usr/lib/*/libecpg.so.6* +usr/share/locale/*/*/ecpglib*.mo diff --git a/debian/libecpg6.symbols b/debian/libecpg6.symbols new file mode 100644 index 00000000000..d20881adac9 --- /dev/null +++ b/debian/libecpg6.symbols @@ -0,0 +1,31 @@ +libecpg.so.6 libecpg6 #MINVER# +* Build-Depends-Package: libecpg-dev + ECPGallocate_desc@Base 0 + ECPGconnect@Base 0 + ECPGdeallocate@Base 0 + ECPGdeallocate_all@Base 0 + ECPGdeallocate_desc@Base 0 + ECPGdebug@Base 0 + ECPGdescribe@Base 0 + ECPGdisconnect@Base 0 + ECPGdo@Base 0 + ECPGdo_descriptor@Base 0 + ECPGfree_auto_mem@Base 0 + ECPGget_PGconn@Base 8.3.1-2~ + ECPGget_desc@Base 0 + ECPGget_desc_header@Base 0 + ECPGget_sqlca@Base 0 + ECPGget_var@Base 9.0~ + ECPGis_noind_null@Base 0 + ECPGprepare@Base 0 + ECPGprepared_statement@Base 0 + ECPGset_desc@Base 0 + ECPGset_desc_header@Base 0 + ECPGset_noind_null@Base 0 + ECPGset_var@Base 9.0~ + ECPGsetcommit@Base 0 + ECPGsetconn@Base 0 + ECPGstatus@Base 0 + ECPGtrans@Base 0 + ECPGtransactionStatus@Base 9.0~ + sqlprint@Base 0 diff --git a/debian/libpgtypes3.install b/debian/libpgtypes3.install new file mode 100644 index 00000000000..decf90ae6bc --- /dev/null +++ b/debian/libpgtypes3.install @@ -0,0 +1 @@ +usr/lib/*/libpgtypes.so.3* diff --git a/debian/libpgtypes3.symbols b/debian/libpgtypes3.symbols new file mode 100644 index 00000000000..a398be3399a --- /dev/null +++ b/debian/libpgtypes3.symbols @@ -0,0 +1,48 @@ +libpgtypes.so.3 libpgtypes3 #MINVER# +* Build-Depends-Package: libecpg-dev + PGTYPESchar_free@Base 11~beta2 + PGTYPESdate_dayofweek@Base 0 + PGTYPESdate_defmt_asc@Base 0 + PGTYPESdate_fmt_asc@Base 0 + PGTYPESdate_free@Base 0 + PGTYPESdate_from_asc@Base 0 + PGTYPESdate_from_timestamp@Base 0 + PGTYPESdate_julmdy@Base 0 + PGTYPESdate_mdyjul@Base 0 + PGTYPESdate_new@Base 0 + PGTYPESdate_to_asc@Base 0 + PGTYPESdate_today@Base 0 + PGTYPESdecimal_free@Base 0 + PGTYPESdecimal_new@Base 0 + PGTYPESinterval_copy@Base 0 + PGTYPESinterval_free@Base 0 + PGTYPESinterval_from_asc@Base 0 + PGTYPESinterval_new@Base 0 + PGTYPESinterval_to_asc@Base 0 + PGTYPESnumeric_add@Base 0 + PGTYPESnumeric_cmp@Base 0 + PGTYPESnumeric_copy@Base 0 + PGTYPESnumeric_div@Base 0 + PGTYPESnumeric_free@Base 0 + PGTYPESnumeric_from_asc@Base 0 + PGTYPESnumeric_from_decimal@Base 0 + PGTYPESnumeric_from_double@Base 0 + PGTYPESnumeric_from_int@Base 0 + PGTYPESnumeric_from_long@Base 0 + PGTYPESnumeric_mul@Base 0 + PGTYPESnumeric_new@Base 0 + PGTYPESnumeric_sub@Base 0 + PGTYPESnumeric_to_asc@Base 0 + PGTYPESnumeric_to_decimal@Base 0 + PGTYPESnumeric_to_double@Base 0 + PGTYPESnumeric_to_int@Base 0 + PGTYPESnumeric_to_long@Base 0 + PGTYPEStimestamp_add_interval@Base 0 + PGTYPEStimestamp_current@Base 0 + PGTYPEStimestamp_defmt_asc@Base 0 + PGTYPEStimestamp_defmt_scan@Base 0 + PGTYPEStimestamp_fmt_asc@Base 0 + PGTYPEStimestamp_from_asc@Base 0 + PGTYPEStimestamp_sub@Base 0 + PGTYPEStimestamp_sub_interval@Base 0 + PGTYPEStimestamp_to_asc@Base 0 diff --git a/debian/libpq-dev.dirs b/debian/libpq-dev.dirs new file mode 100644 index 00000000000..e7724817552 --- /dev/null +++ b/debian/libpq-dev.dirs @@ -0,0 +1 @@ +usr/bin diff --git a/debian/libpq-dev.install b/debian/libpq-dev.install new file mode 100644 index 00000000000..734cb6f3189 --- /dev/null +++ b/debian/libpq-dev.install @@ -0,0 +1,11 @@ +usr/include/postgresql/internal/* +usr/include/postgresql/libpq-events.h +usr/include/postgresql/libpq-fe.h +usr/include/postgresql/libpq/libpq-fs.h +usr/include/postgresql/pg_config*.h +usr/include/postgresql/postgres_ext.h +usr/lib/*/libpq.a +usr/lib/*/libpq.so +usr/lib/*/pkgconfig/libpq.pc +# pg_config manpage for both the perl and C versions +usr/share/postgresql/*/man/man1/pg_config.1* /usr/share/man/man1 diff --git a/debian/libpq5.install b/debian/libpq5.install new file mode 100644 index 00000000000..c9811d7d9f1 --- /dev/null +++ b/debian/libpq5.install @@ -0,0 +1,2 @@ +usr/lib/*/libpq.so.5* +usr/share/locale/*/LC_MESSAGES/libpq*.mo diff --git a/debian/libpq5.symbols b/debian/libpq5.symbols new file mode 100644 index 00000000000..3699673a570 --- /dev/null +++ b/debian/libpq5.symbols @@ -0,0 +1,207 @@ +libpq.so.5 libpq5 #MINVER# +* Build-Depends-Package: libpq-dev + PQbackendPID@Base 0 + PQbinaryTuples@Base 0 + PQcancel@Base 0 + PQcancelBlocking@Base 17~~ + PQcancelCreate@Base 17~~ + PQcancelErrorMessage@Base 17~~ + PQcancelFinish@Base 17~~ + PQcancelPoll@Base 17~~ + PQcancelReset@Base 17~~ + PQcancelSocket@Base 17~~ + PQcancelStart@Base 17~~ + PQcancelStatus@Base 17~~ + PQchangePassword@Base 17~~ + PQclear@Base 0 + PQclientEncoding@Base 0 + PQclosePortal@Base 17~~ + PQclosePrepared@Base 17~~ + PQcmdStatus@Base 0 + PQcmdTuples@Base 0 + PQconndefaults@Base 0 + PQconnectPoll@Base 0 + PQconnectStart@Base 0 + PQconnectStartParams@Base 9.0~ + PQconnectdb@Base 0 + PQconnectdbParams@Base 9.0~ + PQconnectionNeedsPassword@Base 8.3~rc1-1~ + PQconnectionUsedGSSAPI@Base 16~~ + PQconnectionUsedPassword@Base 8.3~ + PQconninfo@Base 9.3~ + PQconninfoFree@Base 0 + PQconninfoParse@Base 8.4~ + PQconsumeInput@Base 0 + PQcopyResult@Base 8.4~ + PQdb@Base 0 + PQdefaultSSLKeyPassHook_OpenSSL@Base 13~~ + PQdescribePortal@Base 0 + PQdescribePrepared@Base 0 + PQdisplayTuples@Base 0 + PQdsplen@Base 0 + PQencryptPassword@Base 0 + PQencryptPasswordConn@Base 10~~ + PQendcopy@Base 0 + PQenterPipelineMode@Base 14~~ + PQenv2encoding@Base 0 + PQerrorMessage@Base 0 + PQescapeBytea@Base 0 + PQescapeByteaConn@Base 0 + PQescapeIdentifier@Base 9.0~ + PQescapeLiteral@Base 9.0~ + PQescapeString@Base 0 + PQescapeStringConn@Base 0 + PQexec@Base 0 + PQexecParams@Base 0 + PQexecPrepared@Base 0 + PQexitPipelineMode@Base 14~~ + PQfformat@Base 0 + PQfinish@Base 0 + PQfireResultCreateEvents@Base 8.4~ + PQflush@Base 0 + PQfmod@Base 0 + PQfn@Base 0 + PQfname@Base 0 + PQfnumber@Base 0 + PQfreeCancel@Base 0 + PQfreeNotify@Base 0 + PQfreemem@Base 0 + PQfsize@Base 0 + PQftable@Base 0 + PQftablecol@Base 0 + PQftype@Base 0 + PQgetCancel@Base 0 + PQgetCopyData@Base 0 + PQgetCurrentTimeUSec@Base 17~beta2 + PQgetResult@Base 0 + PQgetSSLKeyPassHook_OpenSSL@Base 13~~ + PQgetgssctx@Base 12~~ + PQgetisnull@Base 0 + PQgetlength@Base 0 + PQgetline@Base 0 + PQgetlineAsync@Base 0 + PQgetssl@Base 0 + PQgetvalue@Base 0 + PQgssEncInUse@Base 12~~ + PQhost@Base 0 + PQhostaddr@Base 12~~ + PQinitOpenSSL@Base 8.4~ + PQinitSSL@Base 0 + PQinstanceData@Base 8.4~ + PQisBusy@Base 0 + PQisnonblocking@Base 0 + PQisthreadsafe@Base 0 + PQlibVersion@Base 9.1~ + PQmakeEmptyPGresult@Base 0 + PQmblen@Base 0 + PQmblenBounded@Base 14~beta2 + PQnfields@Base 0 + PQnotifies@Base 0 + PQnparams@Base 0 + PQntuples@Base 0 + PQoidStatus@Base 0 + PQoidValue@Base 0 + PQoptions@Base 0 + PQparameterStatus@Base 0 + PQparamtype@Base 0 + PQpass@Base 0 + PQping@Base 9.1~ + PQpingParams@Base 9.1~ + PQpipelineStatus@Base 14~~ + PQpipelineSync@Base 14~~ + PQport@Base 0 + PQprepare@Base 0 + PQprint@Base 0 + PQprintTuples@Base 0 + PQprotocolVersion@Base 0 + PQputCopyData@Base 0 + PQputCopyEnd@Base 0 + PQputline@Base 0 + PQputnbytes@Base 0 + PQregisterEventProc@Base 8.4~ + PQregisterThreadLock@Base 0 + PQrequestCancel@Base 0 + PQresStatus@Base 0 + PQreset@Base 0 + PQresetPoll@Base 0 + PQresetStart@Base 0 + PQresultAlloc@Base 8.4~ + PQresultErrorField@Base 0 + PQresultErrorMessage@Base 0 + PQresultInstanceData@Base 8.4~ + PQresultMemorySize@Base 12~~ + PQresultSetInstanceData@Base 8.4~ + PQresultStatus@Base 0 + PQresultVerboseErrorMessage@Base 9.6~~ + PQsendClosePortal@Base 17~~ + PQsendClosePrepared@Base 17~~ + PQsendDescribePortal@Base 0 + PQsendDescribePrepared@Base 0 + PQsendFlushRequest@Base 15~~ + PQsendPipelineSync@Base 17~~ + PQsendPrepare@Base 0 + PQsendQuery@Base 0 + PQsendQueryParams@Base 0 + PQsendQueryPrepared@Base 0 + PQserverVersion@Base 0 + PQsetChunkedRowsMode@Base 17~~ + PQsetClientEncoding@Base 0 + PQsetErrorContextVisibility@Base 9.6~~ + PQsetErrorVerbosity@Base 0 + PQsetInstanceData@Base 8.4~ + PQsetNoticeProcessor@Base 0 + PQsetNoticeReceiver@Base 0 + PQsetResultAttrs@Base 8.4~ + PQsetSSLKeyPassHook_OpenSSL@Base 13~~ + PQsetSingleRowMode@Base 9.2~beta3 + PQsetTraceFlags@Base 14~beta2 + PQsetdbLogin@Base 0 + PQsetnonblocking@Base 0 + PQsetvalue@Base 8.4~ + PQsocket@Base 0 + PQsocketPoll@Base 17~~ + PQsslAttribute@Base 9.5~~ + PQsslAttributeNames@Base 9.5~~ + PQsslInUse@Base 9.5~~ + PQsslStruct@Base 9.5~~ + PQstatus@Base 0 + PQtrace@Base 0 + PQtransactionStatus@Base 0 + PQtty@Base 0 + PQunescapeBytea@Base 0 + PQuntrace@Base 0 + PQuser@Base 0 + appendBinaryPQExpBuffer@Base 0 + appendPQExpBuffer@Base 0 + appendPQExpBufferChar@Base 0 + appendPQExpBufferStr@Base 0 + createPQExpBuffer@Base 0 + destroyPQExpBuffer@Base 0 + enlargePQExpBuffer@Base 0 + initPQExpBuffer@Base 0 + lo_close@Base 0 + lo_creat@Base 0 + lo_create@Base 0 + lo_export@Base 0 + lo_import@Base 0 + lo_import_with_oid@Base 8.4~ + lo_lseek64@Base 9.3~ + lo_lseek@Base 0 + lo_open@Base 0 + lo_read@Base 0 + lo_tell64@Base 9.3~ + lo_tell@Base 0 + lo_truncate64@Base 9.3~ + lo_truncate@Base 8.3~ + lo_unlink@Base 0 + lo_write@Base 0 + pg_char_to_encoding@Base 0 + pg_encoding_to_char@Base 0 + pg_utf_mblen@Base 0 + pg_valid_server_encoding@Base 0 + pg_valid_server_encoding_id@Base 8.3~beta1-2~ + pgresStatus@Base 0 + pqsignal@Base 0 + printfPQExpBuffer@Base 0 + resetPQExpBuffer@Base 0 + termPQExpBuffer@Base 0 diff --git a/debian/patches/50-per-version-dirs.patch b/debian/patches/50-per-version-dirs.patch new file mode 100644 index 00000000000..8277cdb127b --- /dev/null +++ b/debian/patches/50-per-version-dirs.patch @@ -0,0 +1,29 @@ +Author: Martin Pitt +Description: Use version specific installation directories so that several major versions can be installed in parallel. +Forwarded: No, Debian specific packaging with postgresql-common + + * Install lib files into /usr/lib/postgresql//lib/ + * Install server related header files into /usr/include/postgresql//server/ + +Bug-Debian: http://bugs.debian.org/462037 + +--- a/src/Makefile.global.in ++++ b/src/Makefile.global.in +@@ -119,7 +119,7 @@ libdir := @libdir@ + pkglibdir = $(libdir) + ifeq "$(findstring pgsql, $(pkglibdir))" "" + ifeq "$(findstring postgres, $(pkglibdir))" "" +-override pkglibdir := $(pkglibdir)/postgresql ++override pkglibdir := /usr/lib/postgresql/@PG_MAJORVERSION@/lib + endif + endif + +@@ -167,7 +167,7 @@ endif # PGXS + + # These derived path variables aren't separately configurable. + +-includedir_server = $(pkgincludedir)/server ++includedir_server = $(pkgincludedir)/@PG_MAJORVERSION@/server + includedir_internal = $(pkgincludedir)/internal + pgxsdir = $(pkglibdir)/pgxs + bitcodedir = $(pkglibdir)/bitcode diff --git a/debian/patches/51-default-sockets-in-var.patch b/debian/patches/51-default-sockets-in-var.patch new file mode 100644 index 00000000000..9da49b833a1 --- /dev/null +++ b/debian/patches/51-default-sockets-in-var.patch @@ -0,0 +1,20 @@ +Author: Martin Pitt +Description: Put server Unix sockets into /var/run/postgresql/ by default +Forwarded: No, Debian specific configuration with postgresql-common + +Using /tmp for sockets allows everyone to spoof a PostgreSQL server. Thus use +/var/run/postgresql/ for "system" clusters which run as 'postgres' (user +clusters will still use /tmp). Since system cluster are by far the common case, +set it as default. + +--- a/src/include/pg_config_manual.h ++++ b/src/include/pg_config_manual.h +@@ -206,7 +206,7 @@ + * support them yet. + */ + #ifndef WIN32 +-#define DEFAULT_PGSOCKET_DIR "/tmp" ++#define DEFAULT_PGSOCKET_DIR "/var/run/postgresql" + #else + #define DEFAULT_PGSOCKET_DIR "" + #endif diff --git a/debian/patches/52-tutorial-README.patch b/debian/patches/52-tutorial-README.patch new file mode 100644 index 00000000000..9eb3263c2b3 --- /dev/null +++ b/debian/patches/52-tutorial-README.patch @@ -0,0 +1,16 @@ +Author: Martin Pitt +Description: Update tutorial README for required build dependencies. +Forwarded: No, Debian specific + +--- a/src/tutorial/README ++++ b/src/tutorial/README +@@ -6,8 +6,7 @@ tutorial + This directory contains SQL tutorial scripts. To look at them, first do a + % make + to compile all the scripts and C files for the user-defined functions +-and types. (make needs to be GNU make --- it may be named something +-different on your system, often 'gmake') ++and types. This requires a postgresql-server-dev-* package to be installed. + + Then, run psql with the -s (single-step) flag: + % psql -s diff --git a/debian/patches/53-pg_service.conf_directory_doc.patch b/debian/patches/53-pg_service.conf_directory_doc.patch new file mode 100644 index 00000000000..584b41c01f8 --- /dev/null +++ b/debian/patches/53-pg_service.conf_directory_doc.patch @@ -0,0 +1,19 @@ +Author: Martin Pitt +Description: Update pg_service.conf example to tell the Debian specific file location. +Forwarded: No, Debian specific + +Index: postgresql-9.2-9.2~beta1/src/interfaces/libpq/pg_service.conf.sample +=================================================================== +--- postgresql-9.2-9.2~beta1.orig/src/interfaces/libpq/pg_service.conf.sample 2011-04-27 23:17:22.000000000 +0200 ++++ postgresql-9.2-9.2~beta1/src/interfaces/libpq/pg_service.conf.sample 2011-05-10 11:25:42.151949794 +0200 +@@ -8,8 +8,8 @@ + # to look up such parameters. A sample configuration for postgres is + # included in this file. Lines beginning with '#' are comments. + # +-# Copy this to your sysconf directory (typically /usr/local/pgsql/etc) and +-# rename it pg_service.conf. ++# Copy this to /etc/postgresql-common/ (or select its location with the ++# PGSYSCONFDIR environment variable) and rename it pg_service.conf. + # + # + #[postgres] diff --git a/debian/patches/54-debian-alternatives-for-external-tools.patch b/debian/patches/54-debian-alternatives-for-external-tools.patch new file mode 100644 index 00000000000..0031989718d --- /dev/null +++ b/debian/patches/54-debian-alternatives-for-external-tools.patch @@ -0,0 +1,28 @@ +Author: Martin Pitt +Description: Use Debian alternatives for external tools instead of hardcoded programs +Forwarded: No, Debian specific + +--- a/src/bin/psql/settings.h ++++ b/src/bin/psql/settings.h +@@ -19,8 +19,8 @@ + #define DEFAULT_EDITOR "notepad.exe" + /* no DEFAULT_EDITOR_LINENUMBER_ARG for Notepad */ + #else +-#define DEFAULT_EDITOR "vi" +-#define DEFAULT_EDITOR_LINENUMBER_ARG "+" ++#define DEFAULT_EDITOR "sensible-editor" ++/*#define DEFAULT_EDITOR_LINENUMBER_ARG "+"*/ + #endif + + #define DEFAULT_PROMPT1 "%/%R%x%# " +--- a/src/include/fe_utils/print.h ++++ b/src/include/fe_utils/print.h +@@ -20,7 +20,7 @@ + + /* This is not a particularly great place for this ... */ + #ifndef __CYGWIN__ +-#define DEFAULT_PAGER "more" ++#define DEFAULT_PAGER "pager" + #else + #define DEFAULT_PAGER "less" + #endif diff --git a/debian/patches/70-history b/debian/patches/70-history new file mode 100644 index 00000000000..34c868357f9 --- /dev/null +++ b/debian/patches/70-history @@ -0,0 +1,13 @@ +Author: Christoph Berg +Description: Document Debian location of release notes files. +Forwarded: No, Debian specific + +--- a/HISTORY ++++ b/HISTORY +@@ -3,3 +3,6 @@ + + Distribution file sets include release notes for their version and preceding + versions. Visit the file doc/src/sgml/html/release.html in an HTML browser. ++ ++On Debian systems, the release notes are contained in the postgresql-doc-* ++packages, located in /usr/share/doc/postgresql-doc-*/html/release.html. diff --git a/debian/patches/autoconf2.69 b/debian/patches/autoconf2.69 new file mode 100644 index 00000000000..429044e698b --- /dev/null +++ b/debian/patches/autoconf2.69 @@ -0,0 +1,7 @@ +--- a/configure.ac ++++ b/configure.ac +@@ -22,4 +21,0 @@ AC_INIT([PostgreSQL], [15devel], [pgsql- +-m4_if(m4_defn([m4_PACKAGE_VERSION]), [2.69], [], [m4_fatal([Autoconf version 2.69 is required. +-Untested combinations of 'autoconf' and PostgreSQL versions are not +-recommended. You can remove the check from 'configure.ac' but it is then +-your responsibility whether the result works or not.])]) diff --git a/debian/patches/extension_destdir b/debian/patches/extension_destdir new file mode 100644 index 00000000000..08e7ebc7542 --- /dev/null +++ b/debian/patches/extension_destdir @@ -0,0 +1,228 @@ +--- a/src/backend/commands/extension.c ++++ b/src/backend/commands/extension.c +@@ -392,6 +392,16 @@ get_extension_control_filename(const cha + + get_share_path(my_exec_path, sharepath); + result = (char *) palloc(MAXPGPATH); ++ /* ++ * If extension_destdir is set, try to find the file there first ++ */ ++ if (*extension_destdir != '\0') ++ { ++ snprintf(result, MAXPGPATH, "%s%s/extension/%s.control", ++ extension_destdir, sharepath, extname); ++ if (pg_file_exists(result)) ++ return result; ++ } + snprintf(result, MAXPGPATH, "%s/extension/%s.control", + sharepath, extname); + +@@ -431,6 +441,16 @@ get_extension_aux_control_filename(Exten + scriptdir = get_extension_script_directory(control); + + result = (char *) palloc(MAXPGPATH); ++ /* ++ * If extension_destdir is set, try to find the file there first ++ */ ++ if (*extension_destdir != '\0') ++ { ++ snprintf(result, MAXPGPATH, "%s%s/%s--%s.control", ++ extension_destdir, scriptdir, control->name, version); ++ if (pg_file_exists(result)) ++ return result; ++ } + snprintf(result, MAXPGPATH, "%s/%s--%s.control", + scriptdir, control->name, version); + +@@ -449,6 +469,23 @@ get_extension_script_filename(ExtensionC + scriptdir = get_extension_script_directory(control); + + result = (char *) palloc(MAXPGPATH); ++ /* ++ * If extension_destdir is set, try to find the file there first ++ */ ++ if (*extension_destdir != '\0') ++ { ++ if (from_version) ++ snprintf(result, MAXPGPATH, "%s%s/%s--%s--%s.sql", ++ extension_destdir, scriptdir, control->name, from_version, version); ++ else ++ snprintf(result, MAXPGPATH, "%s%s/%s--%s.sql", ++ extension_destdir, scriptdir, control->name, version); ++ if (pg_file_exists(result)) ++ { ++ pfree(scriptdir); ++ return result; ++ } ++ } + if (from_version) + snprintf(result, MAXPGPATH, "%s/%s--%s--%s.sql", + scriptdir, control->name, from_version, version); +@@ -1208,6 +1245,59 @@ get_ext_ver_list(ExtensionControlFile *c + DIR *dir; + struct dirent *de; + ++ /* ++ * If extension_destdir is set, try to find the files there first ++ */ ++ if (*extension_destdir != '\0') ++ { ++ char location[MAXPGPATH]; ++ ++ snprintf(location, MAXPGPATH, "%s%s", extension_destdir, ++ get_extension_script_directory(control)); ++ dir = AllocateDir(location); ++ while ((de = ReadDir(dir, location)) != NULL) ++ { ++ char *vername; ++ char *vername2; ++ ExtensionVersionInfo *evi; ++ ExtensionVersionInfo *evi2; ++ ++ /* must be a .sql file ... */ ++ if (!is_extension_script_filename(de->d_name)) ++ continue; ++ ++ /* ... matching extension name followed by separator */ ++ if (strncmp(de->d_name, control->name, extnamelen) != 0 || ++ de->d_name[extnamelen] != '-' || ++ de->d_name[extnamelen + 1] != '-') ++ continue; ++ ++ /* extract version name(s) from 'extname--something.sql' filename */ ++ vername = pstrdup(de->d_name + extnamelen + 2); ++ *strrchr(vername, '.') = '\0'; ++ vername2 = strstr(vername, "--"); ++ if (!vername2) ++ { ++ /* It's an install, not update, script; record its version name */ ++ evi = get_ext_ver_info(vername, &evi_list); ++ evi->installable = true; ++ continue; ++ } ++ *vername2 = '\0'; /* terminate first version */ ++ vername2 += 2; /* and point to second */ ++ ++ /* if there's a third --, it's bogus, ignore it */ ++ if (strstr(vername2, "--")) ++ continue; ++ ++ /* Create ExtensionVersionInfos and link them together */ ++ evi = get_ext_ver_info(vername, &evi_list); ++ evi2 = get_ext_ver_info(vername2, &evi_list); ++ evi->reachable = lappend(evi->reachable, evi2); ++ } ++ FreeDir(dir); ++ } ++ + location = get_extension_script_directory(control); + dir = AllocateDir(location); + while ((de = ReadDir(dir, location)) != NULL) +--- a/src/include/utils/guc.h ++++ b/src/include/utils/guc.h +@@ -277,6 +277,7 @@ extern PGDLLIMPORT char *ConfigFileName; + extern PGDLLIMPORT char *HbaFileName; + extern PGDLLIMPORT char *IdentFileName; + extern PGDLLIMPORT char *external_pid_file; ++extern PGDLLIMPORT char *extension_destdir; + + extern PGDLLIMPORT char *application_name; + +--- a/src/backend/utils/fmgr/dfmgr.c ++++ b/src/backend/utils/fmgr/dfmgr.c +@@ -35,6 +35,7 @@ + #include "miscadmin.h" + #include "storage/fd.h" + #include "storage/shmem.h" ++#include "utils/guc.h" + #include "utils/hsearch.h" + + +@@ -415,7 +416,7 @@ expand_dynamic_library_name(const char * + { + bool have_slash; + char *new; +- char *full; ++ char *full, *full2; + + Assert(name); + +@@ -430,6 +431,19 @@ expand_dynamic_library_name(const char * + else + { + full = substitute_libpath_macro(name); ++ /* ++ * If extension_destdir is set, try to find the file there first ++ */ ++ if (*extension_destdir != '\0') ++ { ++ full2 = psprintf("%s%s", extension_destdir, full); ++ if (pg_file_exists(full2)) ++ { ++ pfree(full); ++ return full2; ++ } ++ pfree(full2); ++ } + if (pg_file_exists(full)) + return full; + pfree(full); +@@ -448,6 +462,19 @@ expand_dynamic_library_name(const char * + { + full = substitute_libpath_macro(new); + pfree(new); ++ /* ++ * If extension_destdir is set, try to find the file there first ++ */ ++ if (*extension_destdir != '\0') ++ { ++ full2 = psprintf("%s%s", extension_destdir, full); ++ if (pg_file_exists(full2)) ++ { ++ pfree(full); ++ return full2; ++ } ++ pfree(full2); ++ } + if (pg_file_exists(full)) + return full; + pfree(full); +--- a/src/backend/utils/misc/postgresql.conf.sample ++++ b/src/backend/utils/misc/postgresql.conf.sample +@@ -757,6 +757,8 @@ + # - Other Defaults - + + #dynamic_library_path = '$libdir' ++#extension_destdir = '' # prepend path when loading extensions ++ # and shared objects (added by Debian) + #gin_fuzzy_search_limit = 0 + + +--- a/src/backend/utils/misc/guc_tables.c ++++ b/src/backend/utils/misc/guc_tables.c +@@ -538,6 +538,7 @@ char *ConfigFileName; + char *HbaFileName; + char *IdentFileName; + char *external_pid_file; ++char *extension_destdir; + + char *application_name; + +@@ -4415,6 +4416,17 @@ struct config_string ConfigureNamesStrin + }, + + { ++ {"extension_destdir", PGC_SUSET, FILE_LOCATIONS, ++ gettext_noop("Path to prepend for extension loading."), ++ gettext_noop("This directory is prepended to paths when loading extensions (control and SQL files), and to the '$libdir' directive when loading modules that back functions. The location is made configurable to allow build-time testing of extensions that do not have been installed to their proper location yet."), ++ GUC_SUPERUSER_ONLY ++ }, ++ &extension_destdir, ++ "", ++ NULL, NULL, NULL ++ }, ++ ++ { + {"ssl_library", PGC_INTERNAL, PRESET_OPTIONS, + gettext_noop("Shows the name of the SSL library."), + NULL, diff --git a/debian/patches/filter-debug-prefix-map b/debian/patches/filter-debug-prefix-map new file mode 100644 index 00000000000..08f42b01bd4 --- /dev/null +++ b/debian/patches/filter-debug-prefix-map @@ -0,0 +1,34 @@ +To make the PostgreSQL server packages build reproducibly, we need to remove +the build path from -fdebug-prefix-map and -ffile-prefix-map in CFLAGS. + +* The actual server build still uses the original CFLAGS so the build path is + correctly mapped in the object files. +* The information printed by the pg_config binary and the system view is + filtered in src/common/Makefile and the configure script. +* The build paths stored in Makefile.global are filtered in debian/rules. + (abs_top_builddir, abs_top_srcdir, configure_args, CFLAGS) +* To make PGXS module builds reproducible, pg_buildext copies the environment + CFLAGS to COPT where Makefile.global picks them up, using the prefix maps + from dpkg-buildflags. + +--- a/src/common/Makefile ++++ b/src/common/Makefile +@@ -33,7 +33,7 @@ STD_CPPFLAGS := $(filter-out -I$(top_src + STD_LDFLAGS := $(filter-out -L$(top_builddir)/src/common -L$(top_builddir)/src/port,$(LDFLAGS)) + override CPPFLAGS += -DVAL_CC="\"$(CC)\"" + override CPPFLAGS += -DVAL_CPPFLAGS="\"$(STD_CPPFLAGS)\"" +-override CPPFLAGS += -DVAL_CFLAGS="\"$(CFLAGS)\"" ++override CPPFLAGS += -DVAL_CFLAGS="\"$(filter-out -fdebug-prefix-map=% -ffile-prefix-map=%,$(CFLAGS))\"" + override CPPFLAGS += -DVAL_CFLAGS_SL="\"$(CFLAGS_SL)\"" + override CPPFLAGS += -DVAL_LDFLAGS="\"$(STD_LDFLAGS)\"" + override CPPFLAGS += -DVAL_LDFLAGS_EX="\"$(LDFLAGS_EX)\"" +--- a/configure.ac ++++ b/configure.ac +@@ -27,6 +27,7 @@ AC_COPYRIGHT([Copyright (c) 1996-2023, P + AC_CONFIG_SRCDIR([src/backend/access/common/heaptuple.c]) + AC_CONFIG_AUX_DIR(config) + AC_PREFIX_DEFAULT(/usr/local/pgsql) ++[ac_configure_args=$(echo "$ac_configure_args" | sed -e "s/ -f\(debug\|file\)-prefix-map=[^' ]*//g")] + AC_DEFINE_UNQUOTED(CONFIGURE_ARGS, ["$ac_configure_args"], [Saved arguments from configure]) + + [PG_MAJORVERSION=`expr "$PACKAGE_VERSION" : '\([0-9][0-9]*\)'`] diff --git a/debian/patches/focal-arm64-outline-atomics b/debian/patches/focal-arm64-outline-atomics new file mode 100644 index 00000000000..b87d8f50eb9 --- /dev/null +++ b/debian/patches/focal-arm64-outline-atomics @@ -0,0 +1,23 @@ +Enable outline-atomics on arm64. + +The flag was added in focal's gcc, but is off by default there. It is enabled +by default on all later distributions (hirsute, impish, bullseye, bookwork, +sid). + +https://www.postgresql.org/message-id/flat/1635221042457.21654%40amazon.com + +This patch can be removed once focal is EOL. + +--- a/configure.ac ++++ b/configure.ac +@@ -576,6 +576,10 @@ if test "$GCC" = yes -a "$ICC" = no; the + if test -n "$NOT_THE_CFLAGS"; then + CFLAGS="$CFLAGS -Wno-cast-function-type-strict" + fi ++ if test x"$host_cpu" == x"aarch64"; then ++ PGAC_PROG_CC_CFLAGS_OPT([-moutline-atomics]) ++ PGAC_PROG_CXX_CFLAGS_OPT([-moutline-atomics]) ++ fi + elif test "$ICC" = yes; then + # Intel's compiler has a bug/misoptimization in checking for + # division by NAN (NaN == 0), -mp1 fixes it, so add it to the CFLAGS. diff --git a/debian/patches/hurd-iovec b/debian/patches/hurd-iovec new file mode 100644 index 00000000000..e5255f02230 --- /dev/null +++ b/debian/patches/hurd-iovec @@ -0,0 +1,26 @@ +hurd-i386 does not define IOV_MAX + +--- a/src/include/port/pg_iovec.h ++++ b/src/include/port/pg_iovec.h +@@ -20,9 +20,6 @@ + + #else + +-/* POSIX requires at least 16 as a maximum iovcnt. */ +-#define IOV_MAX 16 +- + /* Define our own POSIX-compatible iovec struct. */ + struct iovec + { +@@ -32,6 +29,11 @@ struct iovec + + #endif + ++/* POSIX requires at least 16 as a maximum iovcnt. */ ++#ifndef IOV_MAX ++#define IOV_MAX 16 ++#endif ++ + /* Define a reasonable maximum that is safe to use on the stack. */ + #define PG_IOV_MAX Min(IOV_MAX, 32) + diff --git a/debian/patches/jit-s390x b/debian/patches/jit-s390x new file mode 100644 index 00000000000..deb64e6af04 --- /dev/null +++ b/debian/patches/jit-s390x @@ -0,0 +1,96 @@ +From 0edaa982336823d4d7af8f10b91579fe0099ef3d Mon Sep 17 00:00:00 2001 +From: Tom Stellard +Date: Tue, 20 Apr 2021 20:14:21 -0700 +Subject: [PATCH] jit: Workaround potential datalayout mismatch on s390x + +LLVM's s390x target uses a different datalayout for z13 and newer processors. +If llvmjit_types.bc is compiled to target a processor older than z13, and +then the JIT runs on a z13 or newer processor, then there will be a mismatch +in datalayouts between llvmjit_types.bc and the JIT engine. This mismatch +causes the JIT to fail at runtime. +--- + src/backend/jit/llvm/llvmjit.c | 46 ++++++++++++++++++++++++++++++++-- + 1 file changed, 44 insertions(+), 2 deletions(-) + +--- a/src/backend/jit/llvm/llvmjit.c ++++ b/src/backend/jit/llvm/llvmjit.c +@@ -777,6 +777,37 @@ llvm_compile_module(LLVMJitContext *cont + } + + /* ++ * For the systemz target, LLVM uses a different datalayout for z13 and newer ++ * CPUs than it does for older CPUs. This can cause a mismatch in datalayouts ++ * in the case where the llvm_types_module is compiled with a pre-z13 CPU ++ * and the JIT is running on z13 or newer. ++ * See computeDataLayout() function in ++ * llvm/lib/Target/SystemZ/SystemZTargetMachine.cpp for information on the ++ * datalayout differences. ++ */ ++static bool ++needs_systemz_workaround(void) ++{ ++ bool ret = false; ++#ifdef __s390x__ ++ LLVMContextRef llvm_context; ++ LLVMTypeRef vec_type; ++ LLVMTargetDataRef llvm_layoutref; ++ if (strncmp(LLVMGetTargetName(llvm_targetref), "systemz", strlen("systemz"))) ++ { ++ return false; ++ } ++ ++ llvm_context = LLVMGetModuleContext(llvm_types_module); ++ vec_type = LLVMVectorType(LLVMIntTypeInContext(llvm_context, 32), 4); ++ llvm_layoutref = LLVMCreateTargetData(llvm_layout); ++ ret = (LLVMABIAlignmentOfType(llvm_layoutref, vec_type) == 16); ++ LLVMDisposeTargetData(llvm_layoutref); ++#endif ++ return ret; ++} ++ ++/* + * Per session initialization. + */ + static void +@@ -785,6 +816,7 @@ llvm_session_initialize(void) + MemoryContext oldcontext; + char *error = NULL; + char *cpu = NULL; ++ char *host_features = NULL; + char *features = NULL; + LLVMTargetMachineRef opt0_tm; + LLVMTargetMachineRef opt3_tm; +@@ -826,10 +858,17 @@ llvm_session_initialize(void) + * features not all CPUs have (weird, huh). + */ + cpu = LLVMGetHostCPUName(); +- features = LLVMGetHostCPUFeatures(); ++ features = host_features = LLVMGetHostCPUFeatures(); + elog(DEBUG2, "LLVMJIT detected CPU \"%s\", with features \"%s\"", + cpu, features); + ++ if (needs_systemz_workaround()) ++ { ++ const char *no_vector =",-vector"; ++ features = malloc(sizeof(char) * (strlen(host_features) + strlen(no_vector) + 1)); ++ sprintf(features, "%s%s", host_features, no_vector); ++ } ++ + opt0_tm = + LLVMCreateTargetMachine(llvm_targetref, llvm_triple, cpu, features, + LLVMCodeGenLevelNone, +@@ -843,8 +882,13 @@ llvm_session_initialize(void) + + LLVMDisposeMessage(cpu); + cpu = NULL; +- LLVMDisposeMessage(features); ++ if (features != host_features) ++ { ++ free(features); ++ } + features = NULL; ++ LLVMDisposeMessage(host_features); ++ host_features = NULL; + + /* force symbols in main binary to be loaded */ + LLVMLoadLibraryPermanently(NULL); diff --git a/debian/patches/libpgport-pkglibdir b/debian/patches/libpgport-pkglibdir new file mode 100644 index 00000000000..d0d3278eafe --- /dev/null +++ b/debian/patches/libpgport-pkglibdir @@ -0,0 +1,82 @@ +Author: Christoph Berg +Description: Move libpgport/libpgcommon/libpgfeutils from libdir to pkglibdir + This allows client applications to link to version-specific libraries. + Used by pg-checksums. +Forwarded: No, (somewhat) Debian specific + +--- a/src/common/Makefile ++++ b/src/common/Makefile +@@ -126,15 +126,15 @@ distprep: kwlist_d.h + + # libpgcommon is needed by some contrib + install: all installdirs +- $(INSTALL_STLIB) libpgcommon.a '$(DESTDIR)$(libdir)/libpgcommon.a' +- $(INSTALL_STLIB) libpgcommon_shlib.a '$(DESTDIR)$(libdir)/libpgcommon_shlib.a' ++ $(INSTALL_STLIB) libpgcommon.a '$(DESTDIR)$(pkglibdir)/libpgcommon.a' ++ $(INSTALL_STLIB) libpgcommon_shlib.a '$(DESTDIR)$(pkglibdir)/libpgcommon_shlib.a' + + installdirs: +- $(MKDIR_P) '$(DESTDIR)$(libdir)' ++ $(MKDIR_P) '$(DESTDIR)$(pkglibdir)' + + uninstall: +- rm -f '$(DESTDIR)$(libdir)/libpgcommon.a' +- rm -f '$(DESTDIR)$(libdir)/libpgcommon_shlib.a' ++ rm -f '$(DESTDIR)$(pkglibdir)/libpgcommon.a' ++ rm -f '$(DESTDIR)$(pkglibdir)/libpgcommon_shlib.a' + + libpgcommon.a: $(OBJS_FRONTEND) + rm -f $@ +--- a/src/fe_utils/Makefile ++++ b/src/fe_utils/Makefile +@@ -52,13 +52,13 @@ distprep: psqlscan.c + + # libpgfeutils could be useful to contrib, so install it + install: all installdirs +- $(INSTALL_STLIB) libpgfeutils.a '$(DESTDIR)$(libdir)/libpgfeutils.a' ++ $(INSTALL_STLIB) libpgfeutils.a '$(DESTDIR)$(pkglibdir)/libpgfeutils.a' + + installdirs: +- $(MKDIR_P) '$(DESTDIR)$(libdir)' ++ $(MKDIR_P) '$(DESTDIR)$(pkglibdir)' + + uninstall: +- rm -f '$(DESTDIR)$(libdir)/libpgfeutils.a' ++ rm -f '$(DESTDIR)$(pkglibdir)/libpgfeutils.a' + + clean distclean: + rm -f libpgfeutils.a $(OBJS) lex.backup +--- a/src/port/Makefile ++++ b/src/port/Makefile +@@ -70,15 +70,15 @@ all: libpgport.a libpgport_shlib.a libpg + + # libpgport is needed by some contrib + install: all installdirs +- $(INSTALL_STLIB) libpgport.a '$(DESTDIR)$(libdir)/libpgport.a' +- $(INSTALL_STLIB) libpgport_shlib.a '$(DESTDIR)$(libdir)/libpgport_shlib.a' ++ $(INSTALL_STLIB) libpgport.a '$(DESTDIR)$(pkglibdir)/libpgport.a' ++ $(INSTALL_STLIB) libpgport_shlib.a '$(DESTDIR)$(pkglibdir)/libpgport_shlib.a' + + installdirs: +- $(MKDIR_P) '$(DESTDIR)$(libdir)' ++ $(MKDIR_P) '$(DESTDIR)$(pkglibdir)' + + uninstall: +- rm -f '$(DESTDIR)$(libdir)/libpgport.a' +- rm -f '$(DESTDIR)$(libdir)/libpgport_shlib.a' ++ rm -f '$(DESTDIR)$(pkglibdir)/libpgport.a' ++ rm -f '$(DESTDIR)$(pkglibdir)/libpgport_shlib.a' + + libpgport.a: $(OBJS) + rm -f $@ +--- a/src/Makefile.global.in ++++ b/src/Makefile.global.in +@@ -608,7 +608,7 @@ libpq_pgport = $(libpq) + # done if they don't, since they will have satisfied all their references + # from these libraries.) + ifdef PGXS +-libpq_pgport_shlib = -L$(libdir) -lpgcommon_shlib -lpgport_shlib $(libpq) ++libpq_pgport_shlib = -L$(pkglibdir) -lpgcommon_shlib -lpgport_shlib $(libpq) + else + libpq_pgport_shlib = -L$(top_builddir)/src/common -lpgcommon_shlib -L$(top_builddir)/src/port -lpgport_shlib $(libpq) + endif diff --git a/debian/patches/pgstat-report-conflicts-immediately.patch b/debian/patches/pgstat-report-conflicts-immediately.patch new file mode 100644 index 00000000000..57603107f76 --- /dev/null +++ b/debian/patches/pgstat-report-conflicts-immediately.patch @@ -0,0 +1,37 @@ +diff --git i/src/backend/utils/activity/pgstat_database.c w/src/backend/utils/activity/pgstat_database.c +index 7149f22f729..bb36d73ec04 100644 +--- i/src/backend/utils/activity/pgstat_database.c ++++ w/src/backend/utils/activity/pgstat_database.c +@@ -81,12 +81,22 @@ void + pgstat_report_recovery_conflict(int reason) + { + PgStat_StatDBEntry *dbentry; ++ PgStat_EntryRef *entry_ref; ++ PgStatShared_Database *sharedent; + + Assert(IsUnderPostmaster); + if (!pgstat_track_counts) + return; + +- dbentry = pgstat_prep_database_pending(MyDatabaseId); ++ /* ++ * Update the shared stats directly - recovery conflicts should never be ++ * common enough for that to be a problem. ++ */ ++ entry_ref = ++ pgstat_get_entry_ref_locked(PGSTAT_KIND_DATABASE, MyDatabaseId, InvalidOid, false); ++ ++ sharedent = (PgStatShared_Database *) entry_ref->shared_stats; ++ dbentry = &sharedent->stats; + + switch (reason) + { +@@ -116,6 +126,8 @@ pgstat_report_recovery_conflict(int reason) + dbentry->conflict_startup_deadlock++; + break; + } ++ ++ pgstat_unlock_entry(entry_ref); + } + + /* diff --git a/debian/patches/series b/debian/patches/series new file mode 100644 index 00000000000..e1346aa1033 --- /dev/null +++ b/debian/patches/series @@ -0,0 +1,14 @@ +50-per-version-dirs.patch +51-default-sockets-in-var.patch +52-tutorial-README.patch +53-pg_service.conf_directory_doc.patch +54-debian-alternatives-for-external-tools.patch +70-history +filter-debug-prefix-map +libpgport-pkglibdir +extension_destdir +autoconf2.69 +focal-arm64-outline-atomics +jit-s390x +hurd-iovec +pgstat-report-conflicts-immediately.patch diff --git a/debian/po/POTFILES.in b/debian/po/POTFILES.in new file mode 100644 index 00000000000..46e50f28c04 --- /dev/null +++ b/debian/po/POTFILES.in @@ -0,0 +1 @@ +[type: gettext/rfc822deb] postgresql-17.templates diff --git a/debian/po/de.po b/debian/po/de.po new file mode 100644 index 00000000000..df2bd4d7012 --- /dev/null +++ b/debian/po/de.po @@ -0,0 +1,37 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER +# This file is distributed under the same license as the postgresql-11 package. +# Copyright (C) Helge Kreutzmann , 2019. +# +msgid "" +msgstr "" +"Project-Id-Version: postgresql-11 11.1-2\n" +"Report-Msgid-Bugs-To: postgresql-11@packages.debian.org\n" +"POT-Creation-Date: 2019-01-09 15:22+0100\n" +"PO-Revision-Date: 2019-01-19 07:33+0100\n" +"Last-Translator: Helge Kreutzmann \n" +"Language-Team: German \n" +"Language: de\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" + +#. Type: boolean +#. Description +#: ../postgresql-11.templates:1001 +msgid "Remove PostgreSQL directories when package is purged?" +msgstr "" +"PostgreSQL-Verzeichnisse entfernen, wenn das Paket endgültig gelöscht wird?" + +#. Type: boolean +#. Description +#: ../postgresql-11.templates:1001 +msgid "" +"Removing the PostgreSQL server package will leave existing database clusters " +"intact, i.e. their configuration, data, and log directories will not be " +"removed. On purging the package, the directories can optionally be removed." +msgstr "" +"Beim Entfernen der PostgreSQL-Server-Pakete werden existierende Datenbank-" +"Cluster intakt gelassen, d.h. ihre Konfigurations-, Daten- und " +"Log-Verzeichnisse werden nicht entfernt. Beim endgültigen Löschen des " +"Pakets können die Verzeichnisse optional entfernt werden." diff --git a/debian/po/es.po b/debian/po/es.po new file mode 100644 index 00000000000..d408e656a0d --- /dev/null +++ b/debian/po/es.po @@ -0,0 +1,58 @@ +# postgresql-13 po-debconf translation to Spanish. +# Copyright (C) 2021 Software in the Public Interest +# This file is distributed under the same license as the postgresql-13 package. +# +# Changes: +# - Initial translation +# Jonathan Bustillos , 2021. +# +# Traductores, si no conocen el formato PO, merece la pena leer la +# documentación de gettext, especialmente las secciones dedicadas a este +# formato, por ejemplo ejecutando: +# info -n '(gettext)PO Files' +# info -n '(gettext)Header Entry' +# +# Equipo de traducción al español, por favor lean antes de traducir +# los siguientes documentos: +# +# - El proyecto de traducción de Debian al español +# http://www.debian.org/intl/spanish/ +# especialmente las notas y normas de traducción en +# http://www.debian.org/intl/spanish/notas +# +# - La guía de traducción de po's de debconf: +# /usr/share/doc/po-debconf/README-trans +# o http://www.debian.org/intl/l10n/po-debconf/README-trans +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2021-03-31 18:37+0000\n" +"PO-Revision-Date: 2021-04-03 14:25-0600\n" +"Last-Translator: Jonathan Bustillos \n" +"Language-Team: Debian Spanish \n" +"Language: \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Plural-Forms: nplurals=2; plural=(n != 1);\n" +"X-Generator: Gtranslator 2.91.7\n" + +#. Type: boolean +#. Description +#: ../postgresql-13.templates:1001 +msgid "Remove PostgreSQL directories when package is purged?" +msgstr "¿Eliminar los directorios de PostgreSQL cuando se purga el paquete?" + +#. Type: boolean +#. Description +#: ../postgresql-13.templates:1001 +msgid "" +"Removing the PostgreSQL server package will leave existing database clusters " +"intact, i.e. their configuration, data, and log directories will not be " +"removed. On purging the package, the directories can optionally be removed." +msgstr "" +"La eliminación del paquete del servidor PostgreSQL dejará intactos los " +"clusters de bases de datos existentes, es decir, no se eliminarán sus " +"directorios de configuración, datos y registro. Al purgar el paquete, los " +"directorios pueden ser eliminados opcionalmente." diff --git a/debian/po/fr.po b/debian/po/fr.po new file mode 100644 index 00000000000..cc3c6ee55c6 --- /dev/null +++ b/debian/po/fr.po @@ -0,0 +1,39 @@ +# Translation of postgresql debconf templates to French +# Copyright (C) 2019 Debian French l10n team +# This file is distributed under the same license as the postgresql-11 package. +# +# Jean-Pierre Giraud , 2019. +msgid "" +msgstr "" +"Project-Id-Version: postgresql-11\n" +"Report-Msgid-Bugs-To: postgresql-11@packages.debian.org\n" +"POT-Creation-Date: 2019-01-09 15:22+0100\n" +"PO-Revision-Date: 2019-01-14 14:25+0100\n" +"Last-Translator: Jean-Pierre Giraud \n" +"Language-Team: French \n" +"Language: fr\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Plural-Forms: nplurals=2; plural=(n > 1);\n" +"X-Generator: Lokalize 2.0\n" + +#. Type: boolean +#. Description +#: ../postgresql-11.templates:1001 +msgid "Remove PostgreSQL directories when package is purged?" +msgstr "" +"Faut-il supprimer les répertoires de PostgreSQL lors de la purge du paquet ?" + +#. Type: boolean +#. Description +#: ../postgresql-11.templates:1001 +msgid "" +"Removing the PostgreSQL server package will leave existing database clusters " +"intact, i.e. their configuration, data, and log directories will not be " +"removed. On purging the package, the directories can optionally be removed." +msgstr "" +"La suppression du paquet du serveur PostgreSQL laissera les grappes de bases " +"de données existantes intactes, c'est-à-dire que leurs répertoires de " +"configuration, de données et de journal ne seront pas supprimés. Lors de la " +"purge du paquet, les répertoires peuvent être supprimés de façon optionnelle." diff --git a/debian/po/it.po b/debian/po/it.po new file mode 100644 index 00000000000..d8020b9fd8b --- /dev/null +++ b/debian/po/it.po @@ -0,0 +1,37 @@ +# postgresql-14 Italian translation. +# Copyright (C) 2022 postgresql-14's copyright holder +# This file is distributed under the same license as the postgresql-14 package. +# Ceppo , 2022. +# +msgid "" +msgstr "" +"Project-Id-Version: postgresql-14\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2022-08-23 19:57+0000\n" +"PO-Revision-Date: 2022-08-23 00:00+0000\n" +"Last-Translator: Ceppo \n" +"Language-Team: Italian \n" +"Language: it\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" + +#. Type: boolean +#. Description +#: ../postgresql-14.templates:1001 +msgid "Remove PostgreSQL directories when package is purged?" +msgstr "Rimuovere le directory di PostgreSQL quando viene eseguito il purge " +"del pacchetto?" + +#. Type: boolean +#. Description +#: ../postgresql-14.templates:1001 +msgid "" +"Removing the PostgreSQL server package will leave existing database clusters " +"intact, i.e. their configuration, data, and log directories will not be " +"removed. On purging the package, the directories can optionally be removed." +msgstr "" +"La rimozione del pacchetto server di PostgreSQL lascerà intatti i cluster " +"di database esistenti, cioè i loro dati, configurazione e directory di log " +"non saranno rimossi. Eseguendo il purge del pacchetto, le directory possono " +"opzionalmente essere rimosse." diff --git a/debian/po/nl.po b/debian/po/nl.po new file mode 100644 index 00000000000..11cda877b96 --- /dev/null +++ b/debian/po/nl.po @@ -0,0 +1,40 @@ +# Dutch translation of postgresql-11 debconf templates. +# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER +# This file is distributed under the same license as the postgresql-11 package. +# FIRST AUTHOR , YEAR. +# Frans Spiesschaert , 2019. +# +msgid "" +msgstr "" +"Project-Id-Version: postgresql-11_11.1-2\n" +"Report-Msgid-Bugs-To: postgresql-11@packages.debian.org\n" +"POT-Creation-Date: 2019-01-09 15:22+0100\n" +"PO-Revision-Date: 2019-01-19 10:43+0100\n" +"Last-Translator: Frans Spiesschaert \n" +"Language-Team: Debian Dutch l10n Team \n" +"Language: nl\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Plural-Forms: nplurals=2; plural=(n != 1);\n" +"X-Generator: Gtranslator 2.91.7\n" + +#. Type: boolean +#. Description +#: ../postgresql-11.templates:1001 +msgid "Remove PostgreSQL directories when package is purged?" +msgstr "" +"De PostgreSQL-mappen verwijderen wanneer het pakket gewist (purged) wordt?" + +#. Type: boolean +#. Description +#: ../postgresql-11.templates:1001 +msgid "" +"Removing the PostgreSQL server package will leave existing database clusters " +"intact, i.e. their configuration, data, and log directories will not be " +"removed. On purging the package, the directories can optionally be removed." +msgstr "" +"Bij het verwijderen van het serverpakket van PostgreSQL blijven de bestaande " +"databaseclusters intact. Dit wil zeggen dat hun configuratie-, gegevens- en " +"logboekmappen niet verwijderd worden. Bij het wissen (purge) van het pakket, " +"kunnen de mappen naar keuze verwijderd worden." diff --git a/debian/po/pt.po b/debian/po/pt.po new file mode 100644 index 00000000000..ee294d15a44 --- /dev/null +++ b/debian/po/pt.po @@ -0,0 +1,39 @@ +# Translation of postgresql-11's debconf messages to European Portuguese +# Copyright (C) 2019 THE postgresql-11'S COPYRIGHT HOLDER +# This file is distributed under the same license as the postgresql-11 package. +# +# Américo Monteiro , 2019. +msgid "" +msgstr "" +"Project-Id-Version: postgresql-11 11.1-2\n" +"Report-Msgid-Bugs-To: postgresql-11@packages.debian.org\n" +"POT-Creation-Date: 2019-01-09 15:22+0100\n" +"PO-Revision-Date: 2019-01-15 00:34+0000\n" +"Last-Translator: Américo Monteiro \n" +"Language-Team: Portuguese <>\n" +"Language: pt\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Plural-Forms: nplurals=2; plural=(n != 1);\n" +"X-Generator: Lokalize 2.0\n" + +#. Type: boolean +#. Description +#: ../postgresql-11.templates:1001 +msgid "Remove PostgreSQL directories when package is purged?" +msgstr "Remover os directórios do PostgreSQL quando o pacote for purgado?" + +#. Type: boolean +#. Description +#: ../postgresql-11.templates:1001 +msgid "" +"Removing the PostgreSQL server package will leave existing database clusters " +"intact, i.e. their configuration, data, and log directories will not be " +"removed. On purging the package, the directories can optionally be removed." +msgstr "" +"Remover o pacote do servidor PostgreSQL irá deixar intactos agrupamentos de " +"bases de dados existentes, isto é, a sua configuração, dados, e relatórios " +"são serão removidos. Ao purgar o pacote, estes directórios podem " +"opcionalmente ser removidos." + diff --git a/debian/po/pt_BR.po b/debian/po/pt_BR.po new file mode 100644 index 00000000000..5e746cf1ca2 --- /dev/null +++ b/debian/po/pt_BR.po @@ -0,0 +1,37 @@ +# Debconf translations for postgresql-11. +# Copyright (C) 2019 THE postgresql-11'S COPYRIGHT HOLDER +# This file is distributed under the same license as the postgresql-11 package. +# Adriano Rafael Gomes , 2019. +# +msgid "" +msgstr "" +"Project-Id-Version: postgresql-11\n" +"Report-Msgid-Bugs-To: postgresql-11@packages.debian.org\n" +"POT-Creation-Date: 2019-01-09 15:22+0100\n" +"PO-Revision-Date: 2019-01-19 18:06-0200\n" +"Last-Translator: Adriano Rafael Gomes \n" +"Language-Team: Brazilian Portuguese \n" +"Language: pt_BR\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" + +#. Type: boolean +#. Description +#: ../postgresql-11.templates:1001 +msgid "Remove PostgreSQL directories when package is purged?" +msgstr "Remover diretórios do PostgreSQL ao expurgar o pacote?" + +#. Type: boolean +#. Description +#: ../postgresql-11.templates:1001 +msgid "" +"Removing the PostgreSQL server package will leave existing database clusters " +"intact, i.e. their configuration, data, and log directories will not be " +"removed. On purging the package, the directories can optionally be removed." +msgstr "" +"Remover o pacote do servidor PostgreSQL deixará os \"clusters\" de bancos de " +"dados existentes intactos, ou seja, suas configurações, dados e diretórios " +"de log não serão removidos. Ao expurgar o pacote, os diretórios podem ser " +"opcionalmente removidos." diff --git a/debian/po/ro.po b/debian/po/ro.po new file mode 100644 index 00000000000..f38fc5581b6 --- /dev/null +++ b/debian/po/ro.po @@ -0,0 +1,50 @@ +# Mesajele în limba română pentru pachetul postgresql. +# translation of postgresql-xx_ro.po to Romanian +# Copyright © 2023 THE PACKAGE'S COPYRIGHT HOLDER +# This file is distributed under the same license as the postgresql package. +# +# Remus-Gabriel Chelu , 2023. +# +# Cronologia traducerii fișierului „postgresql”: +# Traducerea inițială, făcută de R-GC, pentru versiunea postgresql-15_15.1-1. +# Actualizare a traducerii pentru versiunea Y, făcută de X, Y(anul). +# +msgid "" +msgstr "" +"Project-Id-Version: postgresql-15 15.1-1\n" +"Report-Msgid-Bugs-To: postgresql-15@packages.debian.org\n" +"POT-Creation-Date: 2023-01-19 20:42+0000\n" +"PO-Revision-Date: 2023-02-12 17:42+0100\n" +"Last-Translator: Remus-Gabriel Chelu \n" +"Language-Team: Romanian \n" +"Language: ro\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Plural-Forms: nplurals=3; plural=(n==1 ? 0 : n==0 || (n!=1 && n%100>=1 && " +"n%100<=19) ? 1 : 2);\n" +"X-Generator: Poedit 3.2.2\n" + +#. Type: boolean +#. Description +#: ../postgresql-15.templates:1001 +msgid "Remove PostgreSQL directories when package is purged?" +msgstr "Eliminați directoarele PostgreSQL atunci când pachetul este înlăturat?" + +# R-GC, scrie: +# la sugestia lui DȘ, am modificat traducerea +# acestui mesaj, de la: +# „... va lăsa intacte clusterele de baze de date existente, ...”, la: +# „... va lăsa intacte grupurile de servere (clusters) de baze de date existente, ...” +#. Type: boolean +#. Description +#: ../postgresql-15.templates:1001 +msgid "" +"Removing the PostgreSQL server package will leave existing database clusters " +"intact, i.e. their configuration, data, and log directories will not be " +"removed. On purging the package, the directories can optionally be removed." +msgstr "" +"Eliminarea pachetului de server PostgreSQL va lăsa intacte grupurile de servere " +"(clusters) de baze de date existente, adică configurația, datele și " +"directoarele lor de jurnal nu vor fi eliminate. La înlăturarea pachetului, " +"directoarele pot fi eliminate opțional." diff --git a/debian/po/ru.po b/debian/po/ru.po new file mode 100644 index 00000000000..9828439c667 --- /dev/null +++ b/debian/po/ru.po @@ -0,0 +1,39 @@ +# Russian translation of debconf template for postgresql-11 +# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER +# This file is distributed under the same license as the postgresql-11 package. +# Lev Lamberov , 2019 +# +msgid "" +msgstr "" +"Project-Id-Version: postgresql-11\n" +"Report-Msgid-Bugs-To: postgresql-11@packages.debian.org\n" +"POT-Creation-Date: 2019-01-09 15:22+0100\n" +"PO-Revision-Date: 2019-01-27 14:56+0500\n" +"Language-Team: Debian L10N Russian \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"X-Generator: Poedit 2.2.1\n" +"Last-Translator: Lev Lamberov \n" +"Plural-Forms: nplurals=3; plural=(n%10==1 && n%100!=11 ? 0 : n%10>=2 && n" +"%10<=4 && (n%100<12 || n%100>14) ? 1 : 2);\n" +"Language: ru\n" + +#. Type: boolean +#. Description +#: ../postgresql-11.templates:1001 +msgid "Remove PostgreSQL directories when package is purged?" +msgstr "Удалить каталоги PostgreSQL при вычищении пакета?" + +#. Type: boolean +#. Description +#: ../postgresql-11.templates:1001 +msgid "" +"Removing the PostgreSQL server package will leave existing database clusters " +"intact, i.e. their configuration, data, and log directories will not be " +"removed. On purging the package, the directories can optionally be removed." +msgstr "" +"При удалении серверного пакета PostgreSQL существующие кластеры баз данных " +"останутся нетронутыми. То есть, их каталоги с настройками, данными и " +"журналами не будут удалены. При вычистке пакета эти каталоги могут быть при " +"необходимости удалены." diff --git a/debian/po/sv.po b/debian/po/sv.po new file mode 100644 index 00000000000..0f9120fff98 --- /dev/null +++ b/debian/po/sv.po @@ -0,0 +1,36 @@ +# Translation of postgresql-16 debconf template to Swedish +# Copyright (C) 2023 Martin Bagge +# This file is distributed under the same license as the postgresql-16 package. +# +# Martin Bagge , 2023 +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"Report-Msgid-Bugs-To: postgresql-16@packages.debian.org\n" +"POT-Creation-Date: 2023-12-21 17:22+0100\n" +"PO-Revision-Date: 2023-12-20 22:17+0100\n" +"Last-Translator: Martin Bagge \n" +"Language-Team: Swedish \n" +"Language: sv\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" + +#. Type: boolean +#. Description +#: ../postgresql-16.templates:1001 +msgid "Remove PostgreSQL directories when package is purged?" +msgstr "Ska sökvägar för PostgreSQL tas bort när paketet rensas bort helt?" + +#. Type: boolean +#. Description +#: ../postgresql-16.templates:1001 +msgid "" +"Removing the PostgreSQL server package will leave existing database clusters " +"intact, i.e. their configuration, data, and log directories will not be " +"removed. On purging the package, the directories can optionally be removed." +msgstr "" +"När serverpaketet för PostgreSQL tas bort lämnas existerande databaskluster " +"kvar i sin helhet - det vill säga inställnings-, data- och logg-kataloger " +"kommer inte tas bort. När ett paket rensas bort helt (purge) kan katalogerna " +"tas bort automatiskt om så önskas." diff --git a/debian/postgresql-17.install b/debian/postgresql-17.install new file mode 100755 index 00000000000..ddd8d466917 --- /dev/null +++ b/debian/postgresql-17.install @@ -0,0 +1,62 @@ +#!/usr/bin/dh-exec + +usr/lib/postgresql/*/bin/initdb +usr/lib/postgresql/*/bin/oid2name +usr/lib/postgresql/*/bin/pg_archivecleanup +usr/lib/postgresql/*/bin/pgbench +usr/lib/postgresql/*/bin/pg_checksums +usr/lib/postgresql/*/bin/pg_controldata +usr/lib/postgresql/*/bin/pg_createsubscriber +usr/lib/postgresql/*/bin/pg_ctl +usr/lib/postgresql/*/bin/pg_resetwal +usr/lib/postgresql/*/bin/pg_rewind +usr/lib/postgresql/*/bin/pg_test_fsync +usr/lib/postgresql/*/bin/pg_test_timing +usr/lib/postgresql/*/bin/pg_upgrade +usr/lib/postgresql/*/bin/pg_waldump +usr/lib/postgresql/*/bin/pg_walsummary +usr/lib/postgresql/*/bin/postgres +usr/lib/postgresql/*/bin/vacuumlo +[!alpha !hppa !hurd-amd64 !hurd-i386 !ia64 !kfreebsd-amd64 !kfreebsd-i386 !loong64 !m68k !powerpc !riscv64 !sh4 !sparc64 !x32] usr/lib/postgresql/*/lib/bitcode +[!alpha !hppa !hurd-amd64 !hurd-i386 !ia64 !kfreebsd-amd64 !kfreebsd-i386 !loong64 !m68k !powerpc !riscv64 !sh4 !sparc64 !x32] usr/lib/postgresql/*/lib/llvmjit_types.bc +usr/lib/postgresql/*/lib/*.so +usr/share/locale/*/LC_MESSAGES/initdb-*.mo +usr/share/locale/*/LC_MESSAGES/pg_archivecleanup-*.mo +usr/share/locale/*/LC_MESSAGES/pg_checksums-*.mo +usr/share/locale/*/LC_MESSAGES/pg_controldata-*.mo +usr/share/locale/*/LC_MESSAGES/pg_ctl-*.mo +usr/share/locale/*/LC_MESSAGES/pg_resetwal-*.mo +usr/share/locale/*/LC_MESSAGES/pg_rewind-*.mo +usr/share/locale/*/LC_MESSAGES/pg_test_fsync-*.mo +usr/share/locale/*/LC_MESSAGES/pg_test_timing-*.mo +usr/share/locale/*/LC_MESSAGES/pg_upgrade-*.mo +usr/share/locale/*/LC_MESSAGES/pg_waldump-*.mo +usr/share/locale/*/LC_MESSAGES/pg_walsummary-*.mo +usr/share/locale/*/LC_MESSAGES/postgres-*.mo +usr/share/locale/*/LC_MESSAGES/plpgsql-*.mo +[linux-any] usr/share/postgresql/*/contrib/sepgsql.sql +usr/share/postgresql/*/errcodes.txt +usr/share/postgresql/*/extension/* +usr/share/postgresql/*/man/man1/initdb.1* +usr/share/postgresql/*/man/man1/oid2name.1* +usr/share/postgresql/*/man/man1/pg_archivecleanup.1* +usr/share/postgresql/*/man/man1/pgbench.1* +usr/share/postgresql/*/man/man1/pg_checksums.1* +usr/share/postgresql/*/man/man1/pg_controldata.1* +usr/share/postgresql/*/man/man1/pg_createsubscriber.1* +usr/share/postgresql/*/man/man1/pg_ctl.1* +usr/share/postgresql/*/man/man1/pg_resetwal.1* +usr/share/postgresql/*/man/man1/pg_rewind.1* +usr/share/postgresql/*/man/man1/pg_test_fsync.1* +usr/share/postgresql/*/man/man1/pg_test_timing.1* +usr/share/postgresql/*/man/man1/pg_upgrade.1* +usr/share/postgresql/*/man/man1/pg_waldump.1* +usr/share/postgresql/*/man/man1/pg_walsummary.1* +usr/share/postgresql/*/man/man1/postgres.1* +usr/share/postgresql/*/man/man1/vacuumlo.1* +usr/share/postgresql/*/timezonesets/* +usr/share/postgresql/*/tsearch_data +usr/share/postgresql/*/*.sql +usr/share/postgresql/*/*.conf.sample +usr/share/postgresql/*/postgres.bki +usr/share/postgresql/*/sql_features.txt diff --git a/debian/postgresql-17.lintian-overrides b/debian/postgresql-17.lintian-overrides new file mode 100644 index 00000000000..db5908332f9 --- /dev/null +++ b/debian/postgresql-17.lintian-overrides @@ -0,0 +1,20 @@ +# We test for /usr/bin/pg_dropcluster, but run it without path +command-with-path-in-maintainer-script + +# The World's Most Advanced Open Source Relational Database +description-synopsis-starts-with-article + +# We ship binaries and libs in subdirs of /usr/lib/postgresql +executable-in-usr-lib +repeated-path-segment lib * + +# These are PostgreSQL server plugins; some need no external libraries +hardening-no-fortify-functions [usr/lib/postgresql/*/lib/*] +library-not-linked-against-libc [usr/lib/postgresql/*/lib/*] +shared-library-lacks-prerequisites [usr/lib/postgresql/*/lib/*] + +# We use debconf in postrm only +no-debconf-config + +# We store the PostgreSQL catalog version in a custom control field +unknown-field *Postgresql-Catversion diff --git a/debian/postgresql-17.postinst b/debian/postgresql-17.postinst new file mode 100644 index 00000000000..be698435c3f --- /dev/null +++ b/debian/postgresql-17.postinst @@ -0,0 +1,13 @@ +#!/bin/sh + +set -e + +VERSION=${DPKG_MAINTSCRIPT_PACKAGE##*-} + +if [ "$1" = configure ]; then + . /usr/share/postgresql-common/maintscripts-functions + + configure_version $VERSION "$2" +fi + +#DEBHELPER# diff --git a/debian/postgresql-17.postrm b/debian/postgresql-17.postrm new file mode 100644 index 00000000000..c9f88934327 --- /dev/null +++ b/debian/postgresql-17.postrm @@ -0,0 +1,80 @@ +#!/bin/sh + +set -e + +VERSION=${DPKG_MAINTSCRIPT_PACKAGE##*-} + +clean_dir() { + if [ -d "$1" ] && [ ! -L "$1" ]; then + rmdir "$1" >/dev/null 2>/dev/null || true + fi +} + +drop_cluster() { + # if we still have the postgresql-common package, use it to also shutdown + # server, etc.; otherwise just remove the directories + if [ -x /usr/bin/pg_dropcluster ]; then + pg_dropcluster --stop-server $VERSION "$1" + else + # remove data directory + PGDATALINK="/etc/postgresql/$VERSION/$1/pgdata" + if [ -e "$PGDATALINK" ]; then + rm -rf $(readlink -f "$PGDATALINK") "$PGDATALINK" + else + rm -rf "/var/lib/postgresql/$VERSION/$1/" + fi + + # remove log file, including rotated ones + LOGLINK="/etc/postgresql/$VERSION/$1/log" + if [ -e "$LOGLINK" ]; then + LOG=$(readlink -f "$LOGLINK") + rm -f $LOG* "$LOGLINK" + else + rm -f /var/log/postgresql/postgresql-$VERSION-"$1".log* + fi + + # remove conffiles + for f in pg_hba.conf pg_ident.conf postgresql.conf start.conf environment pg_ctl.conf; do + rm -f /etc/postgresql/$VERSION/"$1"/$f + done + # remove empty conf.d directories + for d in /etc/postgresql/$VERSION/"$1"/*/; do + clean_dir "$d" + done + + clean_dir /etc/postgresql/$VERSION/"$1" + fi +} + +purge_package () { + # ask the user if they want to remove clusters. If debconf is not + # available, just remove everything + if [ -e /usr/share/debconf/confmodule ]; then + db_set $DPKG_MAINTSCRIPT_PACKAGE/postrm_purge_data true + db_input high $DPKG_MAINTSCRIPT_PACKAGE/postrm_purge_data || : + db_go || : + db_get $DPKG_MAINTSCRIPT_PACKAGE/postrm_purge_data || : + [ "$RET" = "false" ] && return 0 + fi + + for c in /etc/postgresql/$VERSION/*; do + [ -e "$c/postgresql.conf" ] || continue + cluster=$(basename "$c") + echo "Dropping cluster $cluster..." + drop_cluster "$cluster" + done + + clean_dir /etc/postgresql/$VERSION + clean_dir /var/lib/postgresql/$VERSION + clean_dir /var/log/postgresql/$VERSION +} + +if [ "$1" = purge ] && [ -d "/etc/postgresql/$VERSION" ] && [ "$(ls /etc/postgresql/$VERSION)" ]; then + # can't load debconf from a function + if [ -e /usr/share/debconf/confmodule ]; then + . /usr/share/debconf/confmodule + fi + purge_package +fi + +#DEBHELPER# diff --git a/debian/postgresql-17.preinst b/debian/postgresql-17.preinst new file mode 100644 index 00000000000..a1bccbc9191 --- /dev/null +++ b/debian/postgresql-17.preinst @@ -0,0 +1,18 @@ +#!/bin/sh + +set -e + +MAJOR_VER="${DPKG_MAINTSCRIPT_PACKAGE#postgresql-}" +CATVERSION="@CATVERSION@" # set by override_dh_installdeb + +case $1 in + install|upgrade) + if [ "$2" ]; then + . /usr/share/postgresql-common/maintscripts-functions + preinst_check_catversion "$MAJOR_VER" "$CATVERSION" + fi ;; +esac + +#DEBHELPER# + +exit 0 diff --git a/debian/postgresql-17.prerm b/debian/postgresql-17.prerm new file mode 100644 index 00000000000..828c6fbe9f5 --- /dev/null +++ b/debian/postgresql-17.prerm @@ -0,0 +1,16 @@ +#!/bin/sh + +set -e + +VERSION=${DPKG_MAINTSCRIPT_PACKAGE##*-} + +#DEBHELPER# + +. /usr/share/postgresql-common/maintscripts-functions + +stop_version $VERSION + +if [ "$1" = remove ]; then + remove_version $VERSION +fi + diff --git a/debian/postgresql-17.templates b/debian/postgresql-17.templates new file mode 100644 index 00000000000..21aba5c0903 --- /dev/null +++ b/debian/postgresql-17.templates @@ -0,0 +1,7 @@ +Template: postgresql-17/postrm_purge_data +Type: boolean +Default: true +_Description: Remove PostgreSQL directories when package is purged? + Removing the PostgreSQL server package will leave existing database clusters + intact, i.e. their configuration, data, and log directories will not be + removed. On purging the package, the directories can optionally be removed. diff --git a/debian/postgresql-client-17.install b/debian/postgresql-client-17.install new file mode 100644 index 00000000000..aef5ebf31bc --- /dev/null +++ b/debian/postgresql-client-17.install @@ -0,0 +1,48 @@ +usr/lib/postgresql/*/bin/clusterdb +usr/lib/postgresql/*/bin/createdb +usr/lib/postgresql/*/bin/createuser +usr/lib/postgresql/*/bin/dropdb +usr/lib/postgresql/*/bin/dropuser +usr/lib/postgresql/*/bin/pg_amcheck +usr/lib/postgresql/*/bin/pg_basebackup +usr/lib/postgresql/*/bin/pg_combinebackup +usr/lib/postgresql/*/bin/pg_config +usr/lib/postgresql/*/bin/pg_dump +usr/lib/postgresql/*/bin/pg_dumpall +usr/lib/postgresql/*/bin/pg_isready +usr/lib/postgresql/*/bin/pg_receivewal +usr/lib/postgresql/*/bin/pg_recvlogical +usr/lib/postgresql/*/bin/pg_restore +usr/lib/postgresql/*/bin/pg_verifybackup +usr/lib/postgresql/*/bin/psql +usr/lib/postgresql/*/bin/reindexdb +usr/lib/postgresql/*/bin/vacuumdb +usr/lib/postgresql/*/lib/pgxs/* +usr/share/locale/*/LC_MESSAGES/pg_amcheck-*.mo +usr/share/locale/*/LC_MESSAGES/pg_basebackup-*.mo +usr/share/locale/*/LC_MESSAGES/pg_combinebackup-*.mo +usr/share/locale/*/LC_MESSAGES/pg_config-*.mo +usr/share/locale/*/LC_MESSAGES/pg_dump-*.mo +usr/share/locale/*/LC_MESSAGES/pg_verifybackup-*.mo +usr/share/locale/*/LC_MESSAGES/pgscripts-*.mo +usr/share/locale/*/LC_MESSAGES/psql-*.mo +usr/share/postgresql/*/man/man1/clusterdb.1* +usr/share/postgresql/*/man/man1/createdb.1* +usr/share/postgresql/*/man/man1/createuser.1* +usr/share/postgresql/*/man/man1/dropdb.1* +usr/share/postgresql/*/man/man1/dropuser.1* +usr/share/postgresql/*/man/man1/pg_amcheck.1* +usr/share/postgresql/*/man/man1/pg_basebackup.1* +usr/share/postgresql/*/man/man1/pg_combinebackup.1* +usr/share/postgresql/*/man/man1/pg_dump.1* +usr/share/postgresql/*/man/man1/pg_dumpall.1* +usr/share/postgresql/*/man/man1/pg_isready.1* +usr/share/postgresql/*/man/man1/pg_receivewal.1* +usr/share/postgresql/*/man/man1/pg_recvlogical.1* +usr/share/postgresql/*/man/man1/pg_restore.1* +usr/share/postgresql/*/man/man1/pg_verifybackup.1* +usr/share/postgresql/*/man/man1/psql.1* +usr/share/postgresql/*/man/man1/reindexdb.1* +usr/share/postgresql/*/man/man1/vacuumdb.1* +usr/share/postgresql/*/man/man7/ +usr/share/postgresql/*/psqlrc.sample diff --git a/debian/postgresql-client-17.lintian-overrides b/debian/postgresql-client-17.lintian-overrides new file mode 100644 index 00000000000..a3cef5bfc6c --- /dev/null +++ b/debian/postgresql-client-17.lintian-overrides @@ -0,0 +1,3 @@ +# We ship binaries and libs in subdirs of /usr/lib/postgresql +executable-in-usr-lib +repeated-path-segment lib * diff --git a/debian/postgresql-client-17.postinst b/debian/postgresql-client-17.postinst new file mode 100644 index 00000000000..a5cf251f4f4 --- /dev/null +++ b/debian/postgresql-client-17.postinst @@ -0,0 +1,13 @@ +#!/bin/sh + +set -e + +VERSION=${DPKG_MAINTSCRIPT_PACKAGE##*-} + +. /usr/share/postgresql-common/maintscripts-functions + +if [ "$1" = configure ]; then + configure_client_version $VERSION "$2" +fi + +#DEBHELPER# diff --git a/debian/postgresql-client-17.prerm b/debian/postgresql-client-17.prerm new file mode 100644 index 00000000000..14c21887943 --- /dev/null +++ b/debian/postgresql-client-17.prerm @@ -0,0 +1,12 @@ +#!/bin/sh + +set -e + +VERSION=${DPKG_MAINTSCRIPT_PACKAGE##*-} + +#DEBHELPER# + +if [ "$1" = remove ]; then + . /usr/share/postgresql-common/maintscripts-functions + remove_client_version $VERSION +fi diff --git a/debian/postgresql-doc-17.doc-base b/debian/postgresql-doc-17.doc-base new file mode 100644 index 00000000000..3552aace3c2 --- /dev/null +++ b/debian/postgresql-doc-17.doc-base @@ -0,0 +1,18 @@ +Document: postgresql-17 +Title: PostgreSQL 17 Documentation +Author: The PostgreSQL Global Development Group +Abstract: The documentation for the PostgreSQL database management system, + version 17. PostgreSQL is a powerful, open source object-relational database + system. It is fully ACID compliant, has full support for foreign keys, joins, + views, triggers, and stored procedures (in multiple languages). It includes + most SQL:2008 data types, including INTEGER, NUMERIC, BOOLEAN, CHAR, VARCHAR, + DATE, INTERVAL, and TIMESTAMP. It also supports storage of binary large + objects, including pictures, sounds, or video. It has native programming + interfaces for C/C++, Java, .Net, Perl, Python, Ruby, Tcl, ODBC, among others, + and exceptional documentation. +Section: Data Management + +Format: HTML +Index: /usr/share/doc/postgresql-doc-17/html/index.html +Files: /usr/share/doc/postgresql-doc-17/html/* + diff --git a/debian/postgresql-doc-17.install b/debian/postgresql-doc-17.install new file mode 100644 index 00000000000..a41ef190d3f --- /dev/null +++ b/debian/postgresql-doc-17.install @@ -0,0 +1,2 @@ +usr/share/doc/postgresql-doc-* +usr/share/postgresql/*/man/man3/ diff --git a/debian/postgresql-doc-17.postinst b/debian/postgresql-doc-17.postinst new file mode 100644 index 00000000000..d647cf43e8d --- /dev/null +++ b/debian/postgresql-doc-17.postinst @@ -0,0 +1,30 @@ +#!/bin/sh + +set -e + +VERSION=${DPKG_MAINTSCRIPT_PACKAGE##*-} + +# arguments: version master package [package] +__link_manpages() { + MANS=$(unset GREP_OPTIONS; dpkg -L $3 $4 $5 2>/dev/null | grep -E '/man/.*\.[1-9](\.gz)?$' | grep -v "$2") || true + [ -n "$MANS" ] || return 0 + + SLAVES=$(for i in $MANS; do TARGET=$(echo $i | sed "s/postgresql\/$1\///"); echo -n " --slave $TARGET $(basename $i) $i"; done) + + mkdir -p /usr/share/man/man3 + section=$(echo "$2" | sed -e 's/.*\.\(.*\)\..*/man\1/') + update-alternatives --install /usr/share/man/$section/$2 \ + $2 /usr/share/postgresql/$1/man/$section/$2 \ + ${1}0 $SLAVES +} + +if [ "$1" = configure ]; then + if [ -f /usr/share/postgresql-common/maintscripts-functions ]; then + . /usr/share/postgresql-common/maintscripts-functions + configure_doc_version $VERSION "$2" + else + __link_manpages $VERSION SPI_connect.3.gz "postgresql-doc-$VERSION" + fi +fi + +#DEBHELPER# diff --git a/debian/postgresql-doc-17.prerm b/debian/postgresql-doc-17.prerm new file mode 100644 index 00000000000..cae8f50aba6 --- /dev/null +++ b/debian/postgresql-doc-17.prerm @@ -0,0 +1,18 @@ +#!/bin/sh + +set -e + +VERSION=${DPKG_MAINTSCRIPT_PACKAGE##*-} + +if [ "$1" = remove ]; then + if [ -f /usr/share/postgresql-common/maintscripts-functions ]; then + . /usr/share/postgresql-common/maintscripts-functions + remove_doc_version $VERSION + else + update-alternatives --remove SPI_connect.3.gz \ + /usr/share/postgresql/$VERSION/man/man3/SPI_connect.3.gz + rmdir --ignore-fail-on-non-empty -p /usr/share/man/man1 /usr/share/man/man3 /usr/share/man/man7 + fi +fi + +#DEBHELPER# diff --git a/debian/postgresql-plperl-17.install b/debian/postgresql-plperl-17.install new file mode 100755 index 00000000000..ce5c0e47701 --- /dev/null +++ b/debian/postgresql-plperl-17.install @@ -0,0 +1,6 @@ +#!/usr/bin/dh-exec + +usr/lib/postgresql/*/lib/*plperl*.so +[!alpha !hppa !hurd-amd64 !hurd-i386 !ia64 !kfreebsd-amd64 !kfreebsd-i386 !loong64 !m68k !powerpc !riscv64 !sh4 !sparc64 !x32] usr/lib/postgresql/*/lib/bitcode/*plperl* +usr/share/locale/*/*/plperl-*.mo +usr/share/postgresql/*/extension/*plperl* diff --git a/debian/postgresql-plperl-17.lintian-overrides b/debian/postgresql-plperl-17.lintian-overrides new file mode 120000 index 00000000000..6985719d602 --- /dev/null +++ b/debian/postgresql-plperl-17.lintian-overrides @@ -0,0 +1 @@ +postgresql-17.lintian-overrides \ No newline at end of file diff --git a/debian/postgresql-plpython3-17.install b/debian/postgresql-plpython3-17.install new file mode 100755 index 00000000000..4ff80fe76a6 --- /dev/null +++ b/debian/postgresql-plpython3-17.install @@ -0,0 +1,6 @@ +#!/usr/bin/dh-exec + +usr/lib/postgresql/*/lib/*plpython3*.so +[!alpha !hppa !hurd-amd64 !hurd-i386 !ia64 !kfreebsd-amd64 !kfreebsd-i386 !loong64 !m68k !powerpc !riscv64 !sh4 !sparc64 !x32] usr/lib/postgresql/*/lib/bitcode/*plpython3* +usr/share/locale/*/*/plpython-*.mo +usr/share/postgresql/*/extension/*plpython3* diff --git a/debian/postgresql-plpython3-17.lintian-overrides b/debian/postgresql-plpython3-17.lintian-overrides new file mode 120000 index 00000000000..6985719d602 --- /dev/null +++ b/debian/postgresql-plpython3-17.lintian-overrides @@ -0,0 +1 @@ +postgresql-17.lintian-overrides \ No newline at end of file diff --git a/debian/postgresql-pltcl-17.install b/debian/postgresql-pltcl-17.install new file mode 100644 index 00000000000..f56fc0fb4d4 --- /dev/null +++ b/debian/postgresql-pltcl-17.install @@ -0,0 +1,3 @@ +usr/lib/postgresql/*/lib/pltcl.so +usr/share/locale/*/*/pltcl-*.mo +usr/share/postgresql/*/extension/pltcl* diff --git a/debian/postgresql-pltcl-17.lintian-overrides b/debian/postgresql-pltcl-17.lintian-overrides new file mode 120000 index 00000000000..6985719d602 --- /dev/null +++ b/debian/postgresql-pltcl-17.lintian-overrides @@ -0,0 +1 @@ +postgresql-17.lintian-overrides \ No newline at end of file diff --git a/debian/postgresql-server-dev-17.install b/debian/postgresql-server-dev-17.install new file mode 100644 index 00000000000..0707deee405 --- /dev/null +++ b/debian/postgresql-server-dev-17.install @@ -0,0 +1,5 @@ +usr/include/postgresql/*/server +usr/lib/postgresql/*/bin/pg_bsd_indent +usr/lib/postgresql/*/bin/pgindent +usr/lib/postgresql/*/lib/libpg*.a +usr/share/postgresql/*/typedefs.list diff --git a/debian/postgresql-server-dev-17.lintian-overrides b/debian/postgresql-server-dev-17.lintian-overrides new file mode 120000 index 00000000000..6985719d602 --- /dev/null +++ b/debian/postgresql-server-dev-17.lintian-overrides @@ -0,0 +1 @@ +postgresql-17.lintian-overrides \ No newline at end of file diff --git a/debian/rules b/debian/rules new file mode 100755 index 00000000000..345f1fc71bb --- /dev/null +++ b/debian/rules @@ -0,0 +1,5 @@ +#!/usr/bin/make -f + +MAJOR_VER := 17 + +include /usr/share/postgresql-common/server/postgresql.mk diff --git a/debian/source/format b/debian/source/format new file mode 100644 index 00000000000..d3827e75a5c --- /dev/null +++ b/debian/source/format @@ -0,0 +1 @@ +1.0 diff --git a/debian/source/lintian-overrides b/debian/source/lintian-overrides new file mode 100644 index 00000000000..a244735a805 --- /dev/null +++ b/debian/source/lintian-overrides @@ -0,0 +1,4 @@ +# pregenerated docs contain some tables rendered on a single line +source: source-is-missing [doc/src/sgml/html/*] +# same for these, plus including some regression output files +source: very-long-line-length-in-source-file * diff --git a/debian/tests/Makefile.regress b/debian/tests/Makefile.regress new file mode 100644 index 00000000000..f01a80a52cf --- /dev/null +++ b/debian/tests/Makefile.regress @@ -0,0 +1,5 @@ +MODULE_big = regress +OBJS = regress.o +PG_CONFIG = pg_config +PGXS = $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) diff --git a/debian/tests/control b/debian/tests/control new file mode 100644 index 00000000000..c9db95ff152 --- /dev/null +++ b/debian/tests/control @@ -0,0 +1,21 @@ +Tests: run-testsuite +Depends: + build-essential, + debhelper, + fakeroot, + hunspell-en-us, + iproute2, + locales-all, + logrotate, + netcat-openbsd, + perl, + procps, + @, +Restrictions: needs-root + +Tests: installcheck +Depends: + build-essential, + tzdata, + @, +Restrictions: allow-stderr diff --git a/debian/tests/installcheck b/debian/tests/installcheck new file mode 100755 index 00000000000..fc390ea14a9 --- /dev/null +++ b/debian/tests/installcheck @@ -0,0 +1,41 @@ +#!/bin/sh + +set -eux + +SOURCE=$(dpkg-parsechangelog -SSource) +MAJOR=${SOURCE#*-} +top_srcdir=$PWD + +cd src/test/regress + +# compile regress.so +make -f $top_srcdir/debian/tests/Makefile.regress PG_CONFIG=/usr/lib/postgresql/$MAJOR/bin/pg_config with_llvm=no + +# tell regression files that regress.so is not installed +sed -i -e "s;set regresslib :libdir;set regresslib '$PWD';" sql/* expected/* + +# when root, execute testsuite as user postgres since it insists on wiping the tablespace directory +if [ $(id -u) = 0 ]; then + SU="su postgres" +else + SU="sh" +fi + +$SU < Date: Mon, 26 Feb 2024 21:23:51 +0000 Subject: [PATCH 15/54] Apply deb patch and remove stop version. fix --- HISTORY | 3 + configure.ac | 9 +- debian/postgresql-17.prerm | 2 - src/Makefile.global.in | 6 +- src/backend/commands/extension.c | 90 +++++++++++++++++++ src/backend/jit/llvm/llvmjit.c | 48 +++++++++- src/backend/utils/activity/pgstat_database.c | 14 ++- src/backend/utils/fmgr/dfmgr.c | 29 +++++- src/backend/utils/misc/guc_tables.c | 12 +++ src/backend/utils/misc/postgresql.conf.sample | 2 + src/bin/psql/settings.h | 4 +- src/common/Makefile | 12 +-- src/fe_utils/Makefile | 6 +- src/include/fe_utils/print.h | 2 +- src/include/pg_config_manual.h | 2 +- src/include/utils/guc.h | 1 + src/interfaces/libpq/pg_service.conf.sample | 4 +- src/port/Makefile | 10 +-- src/tutorial/README | 3 +- 19 files changed, 224 insertions(+), 35 deletions(-) diff --git a/HISTORY b/HISTORY index b87be55abf4..f12fd89e960 100644 --- a/HISTORY +++ b/HISTORY @@ -3,3 +3,6 @@ https://www.postgresql.org/docs/current/release.html Distribution file sets include release notes for their version and preceding versions. Visit the file doc/src/sgml/html/release.html in an HTML browser. + +On Debian systems, the release notes are contained in the postgresql-doc-* +packages, located in /usr/share/doc/postgresql-doc-*/html/release.html. diff --git a/configure.ac b/configure.ac index 7bfdc3ec82c..86ed594430a 100644 --- a/configure.ac +++ b/configure.ac @@ -19,14 +19,11 @@ m4_pattern_forbid(^PGAC_)dnl to catch undefined macros AC_INIT([PostgreSQL], [17.9], [pgsql-bugs@lists.postgresql.org], [], [https://www.postgresql.org/]) -m4_if(m4_defn([m4_PACKAGE_VERSION]), [2.69], [], [m4_fatal([Autoconf version 2.69 is required. -Untested combinations of 'autoconf' and PostgreSQL versions are not -recommended. You can remove the check from 'configure.ac' but it is then -your responsibility whether the result works or not.])]) AC_COPYRIGHT([Copyright (c) 1996-2024, PostgreSQL Global Development Group]) AC_CONFIG_SRCDIR([src/backend/access/common/heaptuple.c]) AC_CONFIG_AUX_DIR(config) AC_PREFIX_DEFAULT(/usr/local/pgsql) +[ac_configure_args=$(echo "$ac_configure_args" | sed -e "s/ -f\(debug\|file\)-prefix-map=[^' ]*//g")] AC_DEFINE_UNQUOTED(CONFIGURE_ARGS, ["$ac_configure_args"], [Saved arguments from configure]) [PG_MAJORVERSION=`expr "$PACKAGE_VERSION" : '\([0-9][0-9]*\)'`] @@ -580,6 +577,10 @@ if test "$GCC" = yes -a "$ICC" = no; then if test -n "$NOT_THE_CFLAGS"; then CFLAGS="$CFLAGS -Wno-cast-function-type-strict" fi + if test x"$host_cpu" == x"aarch64"; then + PGAC_PROG_CC_CFLAGS_OPT([-moutline-atomics]) + PGAC_PROG_CXX_CFLAGS_OPT([-moutline-atomics]) + fi elif test "$ICC" = yes; then # Intel's compiler has a bug/misoptimization in checking for # division by NAN (NaN == 0), -mp1 fixes it, so add it to the CFLAGS. diff --git a/debian/postgresql-17.prerm b/debian/postgresql-17.prerm index 828c6fbe9f5..f8d6ac10a20 100644 --- a/debian/postgresql-17.prerm +++ b/debian/postgresql-17.prerm @@ -8,8 +8,6 @@ VERSION=${DPKG_MAINTSCRIPT_PACKAGE##*-} . /usr/share/postgresql-common/maintscripts-functions -stop_version $VERSION - if [ "$1" = remove ]; then remove_version $VERSION fi diff --git a/src/Makefile.global.in b/src/Makefile.global.in index 0c6c31b5bee..78e346ed145 100644 --- a/src/Makefile.global.in +++ b/src/Makefile.global.in @@ -119,7 +119,7 @@ libdir := @libdir@ pkglibdir = $(libdir) ifeq "$(findstring pgsql, $(pkglibdir))" "" ifeq "$(findstring postgres, $(pkglibdir))" "" -override pkglibdir := $(pkglibdir)/postgresql +override pkglibdir := /usr/lib/postgresql/@PG_MAJORVERSION@/lib endif endif @@ -167,7 +167,7 @@ endif # PGXS # These derived path variables aren't separately configurable. -includedir_server = $(pkgincludedir)/server +includedir_server = $(pkgincludedir)/@PG_MAJORVERSION@/server includedir_internal = $(pkgincludedir)/internal pgxsdir = $(pkglibdir)/pgxs bitcodedir = $(pkglibdir)/bitcode @@ -607,7 +607,7 @@ libpq_pgport = $(libpq) # done if they don't, since they will have satisfied all their references # from these libraries.) ifdef PGXS -libpq_pgport_shlib = -L$(libdir) -lpgcommon_shlib -lpgport_shlib $(libpq) +libpq_pgport_shlib = -L$(pkglibdir) -lpgcommon_shlib -lpgport_shlib $(libpq) else libpq_pgport_shlib = -L$(top_builddir)/src/common -lpgcommon_shlib -L$(top_builddir)/src/port -lpgport_shlib $(libpq) endif diff --git a/src/backend/commands/extension.c b/src/backend/commands/extension.c index c427bdcf859..dd6b0bda25d 100644 --- a/src/backend/commands/extension.c +++ b/src/backend/commands/extension.c @@ -523,6 +523,16 @@ get_extension_control_filename(const char *extname) get_share_path(my_exec_path, sharepath); result = (char *) palloc(MAXPGPATH); + /* + * If extension_destdir is set, try to find the file there first + */ + if (*extension_destdir != '\0') + { + snprintf(result, MAXPGPATH, "%s%s/extension/%s.control", + extension_destdir, sharepath, extname); + if (pg_file_exists(result)) + return result; + } snprintf(result, MAXPGPATH, "%s/extension/%s.control", sharepath, extname); @@ -562,6 +572,16 @@ get_extension_aux_control_filename(ExtensionControlFile *control, scriptdir = get_extension_script_directory(control); result = (char *) palloc(MAXPGPATH); + /* + * If extension_destdir is set, try to find the file there first + */ + if (*extension_destdir != '\0') + { + snprintf(result, MAXPGPATH, "%s%s/%s--%s.control", + extension_destdir, scriptdir, control->name, version); + if (pg_file_exists(result)) + return result; + } snprintf(result, MAXPGPATH, "%s/%s--%s.control", scriptdir, control->name, version); @@ -580,6 +600,23 @@ get_extension_script_filename(ExtensionControlFile *control, scriptdir = get_extension_script_directory(control); result = (char *) palloc(MAXPGPATH); + /* + * If extension_destdir is set, try to find the file there first + */ + if (*extension_destdir != '\0') + { + if (from_version) + snprintf(result, MAXPGPATH, "%s%s/%s--%s--%s.sql", + extension_destdir, scriptdir, control->name, from_version, version); + else + snprintf(result, MAXPGPATH, "%s%s/%s--%s.sql", + extension_destdir, scriptdir, control->name, version); + if (pg_file_exists(result)) + { + pfree(scriptdir); + return result; + } + } if (from_version) snprintf(result, MAXPGPATH, "%s/%s--%s--%s.sql", scriptdir, control->name, from_version, version); @@ -1339,6 +1376,59 @@ get_ext_ver_list(ExtensionControlFile *control) DIR *dir; struct dirent *de; + /* + * If extension_destdir is set, try to find the files there first + */ + if (*extension_destdir != '\0') + { + char location[MAXPGPATH]; + + snprintf(location, MAXPGPATH, "%s%s", extension_destdir, + get_extension_script_directory(control)); + dir = AllocateDir(location); + while ((de = ReadDir(dir, location)) != NULL) + { + char *vername; + char *vername2; + ExtensionVersionInfo *evi; + ExtensionVersionInfo *evi2; + + /* must be a .sql file ... */ + if (!is_extension_script_filename(de->d_name)) + continue; + + /* ... matching extension name followed by separator */ + if (strncmp(de->d_name, control->name, extnamelen) != 0 || + de->d_name[extnamelen] != '-' || + de->d_name[extnamelen + 1] != '-') + continue; + + /* extract version name(s) from 'extname--something.sql' filename */ + vername = pstrdup(de->d_name + extnamelen + 2); + *strrchr(vername, '.') = '\0'; + vername2 = strstr(vername, "--"); + if (!vername2) + { + /* It's an install, not update, script; record its version name */ + evi = get_ext_ver_info(vername, &evi_list); + evi->installable = true; + continue; + } + *vername2 = '\0'; /* terminate first version */ + vername2 += 2; /* and point to second */ + + /* if there's a third --, it's bogus, ignore it */ + if (strstr(vername2, "--")) + continue; + + /* Create ExtensionVersionInfos and link them together */ + evi = get_ext_ver_info(vername, &evi_list); + evi2 = get_ext_ver_info(vername2, &evi_list); + evi->reachable = lappend(evi->reachable, evi2); + } + FreeDir(dir); + } + location = get_extension_script_directory(control); dir = AllocateDir(location); while ((de = ReadDir(dir, location)) != NULL) diff --git a/src/backend/jit/llvm/llvmjit.c b/src/backend/jit/llvm/llvmjit.c index a601033fa98..9a420016609 100644 --- a/src/backend/jit/llvm/llvmjit.c +++ b/src/backend/jit/llvm/llvmjit.c @@ -861,6 +861,37 @@ llvm_compile_module(LLVMJitContext *context) errhidecontext(true))); } +/* + * For the systemz target, LLVM uses a different datalayout for z13 and newer + * CPUs than it does for older CPUs. This can cause a mismatch in datalayouts + * in the case where the llvm_types_module is compiled with a pre-z13 CPU + * and the JIT is running on z13 or newer. + * See computeDataLayout() function in + * llvm/lib/Target/SystemZ/SystemZTargetMachine.cpp for information on the + * datalayout differences. + */ +static bool +needs_systemz_workaround(void) +{ + bool ret = false; +#ifdef __s390x__ + LLVMContextRef llvm_context; + LLVMTypeRef vec_type; + LLVMTargetDataRef llvm_layoutref; + if (strncmp(LLVMGetTargetName(llvm_targetref), "systemz", strlen("systemz"))) + { + return false; + } + + llvm_context = LLVMGetModuleContext(llvm_types_module); + vec_type = LLVMVectorType(LLVMIntTypeInContext(llvm_context, 32), 4); + llvm_layoutref = LLVMCreateTargetData(llvm_layout); + ret = (LLVMABIAlignmentOfType(llvm_layoutref, vec_type) == 16); + LLVMDisposeTargetData(llvm_layoutref); +#endif + return ret; +} + /* * Per session initialization. */ @@ -870,6 +901,7 @@ llvm_session_initialize(void) MemoryContext oldcontext; char *error = NULL; char *cpu = NULL; + char *host_features = NULL; char *features = NULL; LLVMTargetMachineRef opt0_tm; LLVMTargetMachineRef opt3_tm; @@ -927,10 +959,17 @@ llvm_session_initialize(void) * features not all CPUs have (weird, huh). */ cpu = LLVMGetHostCPUName(); - features = LLVMGetHostCPUFeatures(); + features = host_features = LLVMGetHostCPUFeatures(); elog(DEBUG2, "LLVMJIT detected CPU \"%s\", with features \"%s\"", cpu, features); + if (needs_systemz_workaround()) + { + const char *no_vector =",-vector"; + features = malloc(sizeof(char) * (strlen(host_features) + strlen(no_vector) + 1)); + sprintf(features, "%s%s", host_features, no_vector); + } + opt0_tm = LLVMCreateTargetMachine(llvm_targetref, llvm_triple, cpu, features, LLVMCodeGenLevelNone, @@ -944,8 +983,13 @@ llvm_session_initialize(void) LLVMDisposeMessage(cpu); cpu = NULL; - LLVMDisposeMessage(features); + if (features != host_features) + { + free(features); + } features = NULL; + LLVMDisposeMessage(host_features); + host_features = NULL; /* force symbols in main binary to be loaded */ LLVMLoadLibraryPermanently(NULL); diff --git a/src/backend/utils/activity/pgstat_database.c b/src/backend/utils/activity/pgstat_database.c index 3398a798460..1a0a61e07a9 100644 --- a/src/backend/utils/activity/pgstat_database.c +++ b/src/backend/utils/activity/pgstat_database.c @@ -81,12 +81,22 @@ void pgstat_report_recovery_conflict(int reason) { PgStat_StatDBEntry *dbentry; + PgStat_EntryRef *entry_ref; + PgStatShared_Database *sharedent; Assert(IsUnderPostmaster); if (!pgstat_track_counts) return; - dbentry = pgstat_prep_database_pending(MyDatabaseId); + /* + * Update the shared stats directly - recovery conflicts should never be + * common enough for that to be a problem. + */ + entry_ref = + pgstat_get_entry_ref_locked(PGSTAT_KIND_DATABASE, MyDatabaseId, InvalidOid, false); + + sharedent = (PgStatShared_Database *) entry_ref->shared_stats; + dbentry = &sharedent->stats; switch (reason) { @@ -116,6 +126,8 @@ pgstat_report_recovery_conflict(int reason) dbentry->conflict_startup_deadlock++; break; } + + pgstat_unlock_entry(entry_ref); } /* diff --git a/src/backend/utils/fmgr/dfmgr.c b/src/backend/utils/fmgr/dfmgr.c index 092004dcf3b..25971b25b61 100644 --- a/src/backend/utils/fmgr/dfmgr.c +++ b/src/backend/utils/fmgr/dfmgr.c @@ -35,6 +35,7 @@ #include "miscadmin.h" #include "storage/fd.h" #include "storage/shmem.h" +#include "utils/guc.h" #include "utils/hsearch.h" @@ -415,7 +416,7 @@ expand_dynamic_library_name(const char *name) { bool have_slash; char *new; - char *full; + char *full, *full2; Assert(name); @@ -430,6 +431,19 @@ expand_dynamic_library_name(const char *name) else { full = substitute_libpath_macro(name); + /* + * If extension_destdir is set, try to find the file there first + */ + if (*extension_destdir != '\0') + { + full2 = psprintf("%s%s", extension_destdir, full); + if (pg_file_exists(full2)) + { + pfree(full); + return full2; + } + pfree(full2); + } if (pg_file_exists(full)) return full; pfree(full); @@ -448,6 +462,19 @@ expand_dynamic_library_name(const char *name) { full = substitute_libpath_macro(new); pfree(new); + /* + * If extension_destdir is set, try to find the file there first + */ + if (*extension_destdir != '\0') + { + full2 = psprintf("%s%s", extension_destdir, full); + if (pg_file_exists(full2)) + { + pfree(full); + return full2; + } + pfree(full2); + } if (pg_file_exists(full)) return full; pfree(full); diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index 6eccfa85428..a79b3cd61cd 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -550,6 +550,7 @@ char *ConfigFileName; char *HbaFileName; char *IdentFileName; char *external_pid_file; +char *extension_destdir; char *application_name; @@ -4502,6 +4503,17 @@ struct config_string ConfigureNamesString[] = check_canonical_path, NULL, NULL }, + { + {"extension_destdir", PGC_SUSET, FILE_LOCATIONS, + gettext_noop("Path to prepend for extension loading."), + gettext_noop("This directory is prepended to paths when loading extensions (control and SQL files), and to the '$libdir' directive when loading modules that back functions. The location is made configurable to allow build-time testing of extensions that do not have been installed to their proper location yet."), + GUC_SUPERUSER_ONLY + }, + &extension_destdir, + "", + NULL, NULL, NULL + }, + { {"ssl_library", PGC_INTERNAL, PRESET_OPTIONS, gettext_noop("Shows the name of the SSL library."), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index 6c761f83165..e731b661c6e 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -776,6 +776,8 @@ # - Other Defaults - #dynamic_library_path = '$libdir' +#extension_destdir = '' # prepend path when loading extensions + # and shared objects (added by Debian) #gin_fuzzy_search_limit = 0 diff --git a/src/bin/psql/settings.h b/src/bin/psql/settings.h index 505f99d8e47..52e910c005f 100644 --- a/src/bin/psql/settings.h +++ b/src/bin/psql/settings.h @@ -19,8 +19,8 @@ #define DEFAULT_EDITOR "notepad.exe" /* no DEFAULT_EDITOR_LINENUMBER_ARG for Notepad */ #else -#define DEFAULT_EDITOR "vi" -#define DEFAULT_EDITOR_LINENUMBER_ARG "+" +#define DEFAULT_EDITOR "sensible-editor" +/*#define DEFAULT_EDITOR_LINENUMBER_ARG "+"*/ #endif #define DEFAULT_PROMPT1 "%/%R%x%# " diff --git a/src/common/Makefile b/src/common/Makefile index 3d83299432b..a1b90c994aa 100644 --- a/src/common/Makefile +++ b/src/common/Makefile @@ -33,7 +33,7 @@ STD_CPPFLAGS := $(filter-out -I$(top_srcdir)/src/include -I$(top_builddir)/src/i STD_LDFLAGS := $(filter-out -L$(top_builddir)/src/common -L$(top_builddir)/src/port,$(LDFLAGS)) override CPPFLAGS += -DVAL_CC="\"$(CC)\"" override CPPFLAGS += -DVAL_CPPFLAGS="\"$(STD_CPPFLAGS)\"" -override CPPFLAGS += -DVAL_CFLAGS="\"$(CFLAGS)\"" +override CPPFLAGS += -DVAL_CFLAGS="\"$(filter-out -fdebug-prefix-map=% -ffile-prefix-map=%,$(CFLAGS))\"" override CPPFLAGS += -DVAL_CFLAGS_SL="\"$(CFLAGS_SL)\"" override CPPFLAGS += -DVAL_LDFLAGS="\"$(STD_LDFLAGS)\"" override CPPFLAGS += -DVAL_LDFLAGS_EX="\"$(LDFLAGS_EX)\"" @@ -125,15 +125,15 @@ all: libpgcommon.a libpgcommon_shlib.a libpgcommon_srv.a # libpgcommon is needed by some contrib install: all installdirs - $(INSTALL_STLIB) libpgcommon.a '$(DESTDIR)$(libdir)/libpgcommon.a' - $(INSTALL_STLIB) libpgcommon_shlib.a '$(DESTDIR)$(libdir)/libpgcommon_shlib.a' + $(INSTALL_STLIB) libpgcommon.a '$(DESTDIR)$(pkglibdir)/libpgcommon.a' + $(INSTALL_STLIB) libpgcommon_shlib.a '$(DESTDIR)$(pkglibdir)/libpgcommon_shlib.a' installdirs: - $(MKDIR_P) '$(DESTDIR)$(libdir)' + $(MKDIR_P) '$(DESTDIR)$(pkglibdir)' uninstall: - rm -f '$(DESTDIR)$(libdir)/libpgcommon.a' - rm -f '$(DESTDIR)$(libdir)/libpgcommon_shlib.a' + rm -f '$(DESTDIR)$(pkglibdir)/libpgcommon.a' + rm -f '$(DESTDIR)$(pkglibdir)/libpgcommon_shlib.a' libpgcommon.a: $(OBJS_FRONTEND) rm -f $@ diff --git a/src/fe_utils/Makefile b/src/fe_utils/Makefile index 946c05258f0..92b80f586d0 100644 --- a/src/fe_utils/Makefile +++ b/src/fe_utils/Makefile @@ -50,13 +50,13 @@ psqlscan.c: FLEX_FIX_WARNING=yes # libpgfeutils could be useful to contrib, so install it install: all installdirs - $(INSTALL_STLIB) libpgfeutils.a '$(DESTDIR)$(libdir)/libpgfeutils.a' + $(INSTALL_STLIB) libpgfeutils.a '$(DESTDIR)$(pkglibdir)/libpgfeutils.a' installdirs: - $(MKDIR_P) '$(DESTDIR)$(libdir)' + $(MKDIR_P) '$(DESTDIR)$(pkglibdir)' uninstall: - rm -f '$(DESTDIR)$(libdir)/libpgfeutils.a' + rm -f '$(DESTDIR)$(pkglibdir)/libpgfeutils.a' clean distclean: rm -f libpgfeutils.a $(OBJS) lex.backup diff --git a/src/include/fe_utils/print.h b/src/include/fe_utils/print.h index 72824c5c2fa..354fd585d28 100644 --- a/src/include/fe_utils/print.h +++ b/src/include/fe_utils/print.h @@ -20,7 +20,7 @@ /* This is not a particularly great place for this ... */ #ifndef __CYGWIN__ -#define DEFAULT_PAGER "more" +#define DEFAULT_PAGER "pager" #else #define DEFAULT_PAGER "less" #endif diff --git a/src/include/pg_config_manual.h b/src/include/pg_config_manual.h index f941ee2faf8..96a8f2c87c6 100644 --- a/src/include/pg_config_manual.h +++ b/src/include/pg_config_manual.h @@ -206,7 +206,7 @@ * support them yet. */ #ifndef WIN32 -#define DEFAULT_PGSOCKET_DIR "/tmp" +#define DEFAULT_PGSOCKET_DIR "/var/run/postgresql" #else #define DEFAULT_PGSOCKET_DIR "" #endif diff --git a/src/include/utils/guc.h b/src/include/utils/guc.h index 946a63015a1..d09544f64a4 100644 --- a/src/include/utils/guc.h +++ b/src/include/utils/guc.h @@ -278,6 +278,7 @@ extern PGDLLIMPORT char *ConfigFileName; extern PGDLLIMPORT char *HbaFileName; extern PGDLLIMPORT char *IdentFileName; extern PGDLLIMPORT char *external_pid_file; +extern PGDLLIMPORT char *extension_destdir; extern PGDLLIMPORT char *application_name; diff --git a/src/interfaces/libpq/pg_service.conf.sample b/src/interfaces/libpq/pg_service.conf.sample index 5a1c083538b..7ef2ebde701 100644 --- a/src/interfaces/libpq/pg_service.conf.sample +++ b/src/interfaces/libpq/pg_service.conf.sample @@ -8,8 +8,8 @@ # to look up such parameters. A sample configuration for postgres is # included in this file. Lines beginning with '#' are comments. # -# Copy this to your sysconf directory (typically /usr/local/pgsql/etc) and -# rename it pg_service.conf. +# Copy this to /etc/postgresql-common/ (or select its location with the +# PGSYSCONFDIR environment variable) and rename it pg_service.conf. # # #[postgres] diff --git a/src/port/Makefile b/src/port/Makefile index db7c02117b0..fc5f629e5aa 100644 --- a/src/port/Makefile +++ b/src/port/Makefile @@ -69,15 +69,15 @@ all: libpgport.a libpgport_shlib.a libpgport_srv.a # libpgport is needed by some contrib install: all installdirs - $(INSTALL_STLIB) libpgport.a '$(DESTDIR)$(libdir)/libpgport.a' - $(INSTALL_STLIB) libpgport_shlib.a '$(DESTDIR)$(libdir)/libpgport_shlib.a' + $(INSTALL_STLIB) libpgport.a '$(DESTDIR)$(pkglibdir)/libpgport.a' + $(INSTALL_STLIB) libpgport_shlib.a '$(DESTDIR)$(pkglibdir)/libpgport_shlib.a' installdirs: - $(MKDIR_P) '$(DESTDIR)$(libdir)' + $(MKDIR_P) '$(DESTDIR)$(pkglibdir)' uninstall: - rm -f '$(DESTDIR)$(libdir)/libpgport.a' - rm -f '$(DESTDIR)$(libdir)/libpgport_shlib.a' + rm -f '$(DESTDIR)$(pkglibdir)/libpgport.a' + rm -f '$(DESTDIR)$(pkglibdir)/libpgport_shlib.a' libpgport.a: $(OBJS) rm -f $@ diff --git a/src/tutorial/README b/src/tutorial/README index b137cdfad34..be070c573db 100644 --- a/src/tutorial/README +++ b/src/tutorial/README @@ -6,8 +6,7 @@ tutorial This directory contains SQL tutorial scripts. To look at them, first do a % make to compile all the scripts and C files for the user-defined functions -and types. (make needs to be GNU make --- it may be named something -different on your system, often 'gmake') +and types. This requires a postgresql-server-dev-* package to be installed. Then, run psql with the -s (single-step) flag: % psql -s From de3daf99ac086e3cc0af3bafdb52576458b7e405 Mon Sep 17 00:00:00 2001 From: Kirill Reshke Date: Tue, 27 Feb 2024 00:29:43 +0300 Subject: [PATCH 16/54] Yandex build infra commit --- Dockerfile | 53 ++++++++++++++++++++++++++++++++++++++++++++ docker/entrypoint.sh | 22 ++++++++++++++++++ docker/tzdata.sh | 9 ++++++++ prepare-build.sh | 17 ++++++++++++++ 4 files changed, 101 insertions(+) create mode 100644 Dockerfile create mode 100755 docker/entrypoint.sh create mode 100755 docker/tzdata.sh create mode 100755 prepare-build.sh diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000000..76faa3a2045 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,53 @@ +ARG codename +FROM ubuntu:${codename:-bionic} + +ARG codename +ENV CODE_NAME=${codename:-bionic} + +ARG pgdg +ENV PGDG_VER=${pgdg:-242-2-pgdg18.04+1+yandex220} + +ENV DEBIAN_FRONTEND=noninteractive +ENV TZ=Europe/Moskow +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + +RUN sed -i 's/archive.ubuntu.com/mirror.yandex.ru/g' /etc/apt/sources.list &&\ + apt-get update && apt-get install -y --no-install-recommends \ + sudo build-essential \ + gcc lsb-release libssl-dev gnupg openssl \ + gdb git curl + +RUN echo "deb http://dist.yandex.ru/mdb-${CODE_NAME}-secure stable/all/" >> /etc/apt/sources.list +RUN echo "deb http://dist.yandex.ru/mdb-${CODE_NAME}-secure stable/\$(ARCH)/" >> /etc/apt/sources.list + +RUN curl -s 'http://keyserver.ubuntu.com/pks/lookup?op=get&search=0xafc3ce0d00e3c45a357e9e637fcd11186050cd1a' | \ + gpg --dearmour -o /etc/apt/trusted.gpg.d/yandex.gpg + +RUN apt-get update && apt-get install -y --no-install-recommends \ + sudo build-essential \ + gcc lsb-release libssl-dev gnupg openssl \ + gdb git \ + libpam0g-dev \ + debhelper debootstrap devscripts make equivs debhelper-compat \ + libz-dev flex libicu-dev libio-pty-perl libipc-run-perl libkrb5-dev \ + libldap2-dev liblz4-dev liblz4-tool zstd libperl-dev libreadline-dev libselinux1-dev llvm-dev \ + libsystemd-dev libxml2-dev libxml2-utils libxslt1-dev \ + pkg-config python3-dev systemtap-sdt-dev tcl-dev uuid-dev xsltproc zlib1g-dev \ + bison dh-exec docbook-xml docbook-xsl + +RUN apt-get install -y \ + libmdblocales1 libmdblocales-dev \ + postgresql-client-common=${PGDG_VER} \ + postgresql-common=${PGDG_VER} + +RUN groupadd -g 999 build-user && \ + useradd -r -u 999 -g build-user build-user + +COPY . /home/build-user +RUN chown build-user:build-user /home -R && usermod -aG sudo build-user + +RUN echo 'build-user ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers + +USER build-user + +ENTRYPOINT ["/home/build-user/docker/entrypoint.sh"] diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh new file mode 100755 index 00000000000..3c27e80a4f5 --- /dev/null +++ b/docker/entrypoint.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -ex + +export DEBIAN_FRONTEND=noninteractive +export TZ=Europe/Moskow +sudo bash -c "echo $TZ > /etc/timezone" + +cd /home/build-user + +sudo ./docker/tzdata.sh + +cat debian/changelog +#export DEB_BUILD_OPTIONS="nocheck" + +sudo mk-build-deps --build-dep --install --tool='apt-get -o Debug::pkgProblemResolver=yes --no-install-recommends --yes' debian/control + +dpkg-buildpackage -b -rfakeroot -us -uc +#dpkg-buildpackage -us -uc + +cd /home +rm -fr build-user + diff --git a/docker/tzdata.sh b/docker/tzdata.sh new file mode 100755 index 00000000000..004e6de4101 --- /dev/null +++ b/docker/tzdata.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +export DEBIAN_FRONTEND=noninteractive +#install tzdata package +apt-get install -y tzdata +# set your timezone +ln -fs /usr/share/zoneinfo/$TZ /etc/localtime +dpkg-reconfigure --frontend noninteractive tzdata + diff --git a/prepare-build.sh b/prepare-build.sh new file mode 100755 index 00000000000..fbce2f8aa99 --- /dev/null +++ b/prepare-build.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +export PACKAGE_NAME=postgresql-17 +export BUILD_USER=mdb-cc +export VERSION=$(grep 'PACKAGE_VERSION=' configure | cut -d= -f2 | sed s/\'//g)-201-yandex.$(git rev-list HEAD --count).$(git rev-parse --short HEAD) +export LC_ALL=C + +cat > debian/changelog< $(date +'%a, %d %b %Y %H:%M:%S %z') +EOH + +echo "VERSION=$VERSION" > version.properties + From d310a8448b4e2d63a5c4dc3ea95bfe3e762c687f Mon Sep 17 00:00:00 2001 From: Kirill Reshke Date: Tue, 26 Dec 2023 14:43:49 +0300 Subject: [PATCH 17/54] Untrust all contrib --- contrib/bool_plperl/bool_plperl.control | 2 +- contrib/btree_gin/btree_gin.control | 2 +- contrib/btree_gist/btree_gist.control | 2 +- contrib/citext/citext.control | 2 +- contrib/cube/cube.control | 2 +- contrib/dict_int/dict_int.control | 2 +- contrib/fuzzystrmatch/fuzzystrmatch.control | 2 +- contrib/hstore/hstore.control | 2 +- contrib/intarray/intarray.control | 2 +- contrib/isn/isn.control | 2 +- contrib/jsonb_plperl/jsonb_plperl.control | 2 +- contrib/lo/lo.control | 2 +- contrib/ltree/ltree.control | 2 +- contrib/pg_trgm/pg_trgm.control | 2 +- contrib/pgcrypto/pgcrypto.control | 2 +- contrib/seg/seg.control | 2 +- contrib/tablefunc/tablefunc.control | 2 +- contrib/tcn/tcn.control | 2 +- contrib/tsm_system_rows/tsm_system_rows.control | 2 +- contrib/tsm_system_time/tsm_system_time.control | 2 +- contrib/unaccent/unaccent.control | 2 +- contrib/uuid-ossp/uuid-ossp.control | 2 +- 22 files changed, 22 insertions(+), 22 deletions(-) diff --git a/contrib/bool_plperl/bool_plperl.control b/contrib/bool_plperl/bool_plperl.control index af3e6b1966f..65f9755adad 100644 --- a/contrib/bool_plperl/bool_plperl.control +++ b/contrib/bool_plperl/bool_plperl.control @@ -3,5 +3,5 @@ comment = 'transform between bool and plperl' default_version = '1.0' module_pathname = '$libdir/bool_plperl' relocatable = true -trusted = true +trusted = false requires = 'plperl' diff --git a/contrib/btree_gin/btree_gin.control b/contrib/btree_gin/btree_gin.control index 67d0c997d8d..5fc20d2cd65 100644 --- a/contrib/btree_gin/btree_gin.control +++ b/contrib/btree_gin/btree_gin.control @@ -3,4 +3,4 @@ comment = 'support for indexing common datatypes in GIN' default_version = '1.3' module_pathname = '$libdir/btree_gin' relocatable = true -trusted = true +trusted = false diff --git a/contrib/btree_gist/btree_gist.control b/contrib/btree_gist/btree_gist.control index fa9171a80a2..b3467f6a071 100644 --- a/contrib/btree_gist/btree_gist.control +++ b/contrib/btree_gist/btree_gist.control @@ -3,4 +3,4 @@ comment = 'support for indexing common datatypes in GiST' default_version = '1.7' module_pathname = '$libdir/btree_gist' relocatable = true -trusted = true +trusted = false diff --git a/contrib/citext/citext.control b/contrib/citext/citext.control index ccf445475d0..0934dec1098 100644 --- a/contrib/citext/citext.control +++ b/contrib/citext/citext.control @@ -3,4 +3,4 @@ comment = 'data type for case-insensitive character strings' default_version = '1.6' module_pathname = '$libdir/citext' relocatable = true -trusted = true +trusted = false diff --git a/contrib/cube/cube.control b/contrib/cube/cube.control index 50427ec1170..cf21f9e5fd9 100644 --- a/contrib/cube/cube.control +++ b/contrib/cube/cube.control @@ -3,4 +3,4 @@ comment = 'data type for multidimensional cubes' default_version = '1.5' module_pathname = '$libdir/cube' relocatable = true -trusted = true +trusted = false diff --git a/contrib/dict_int/dict_int.control b/contrib/dict_int/dict_int.control index ec04ccea91a..1ca11400d41 100644 --- a/contrib/dict_int/dict_int.control +++ b/contrib/dict_int/dict_int.control @@ -3,4 +3,4 @@ comment = 'text search dictionary template for integers' default_version = '1.0' module_pathname = '$libdir/dict_int' relocatable = true -trusted = true +trusted = false diff --git a/contrib/fuzzystrmatch/fuzzystrmatch.control b/contrib/fuzzystrmatch/fuzzystrmatch.control index 8b6e9fd9935..e0ffadcbddb 100644 --- a/contrib/fuzzystrmatch/fuzzystrmatch.control +++ b/contrib/fuzzystrmatch/fuzzystrmatch.control @@ -3,4 +3,4 @@ comment = 'determine similarities and distance between strings' default_version = '1.2' module_pathname = '$libdir/fuzzystrmatch' relocatable = true -trusted = true +trusted = false diff --git a/contrib/hstore/hstore.control b/contrib/hstore/hstore.control index 89e3c746c46..248223b9558 100644 --- a/contrib/hstore/hstore.control +++ b/contrib/hstore/hstore.control @@ -3,4 +3,4 @@ comment = 'data type for storing sets of (key, value) pairs' default_version = '1.8' module_pathname = '$libdir/hstore' relocatable = true -trusted = true +trusted = false diff --git a/contrib/intarray/intarray.control b/contrib/intarray/intarray.control index c3ff753e2cf..e01110f2681 100644 --- a/contrib/intarray/intarray.control +++ b/contrib/intarray/intarray.control @@ -3,4 +3,4 @@ comment = 'functions, operators, and index support for 1-D arrays of integers' default_version = '1.5' module_pathname = '$libdir/_int' relocatable = true -trusted = true +trusted = false diff --git a/contrib/isn/isn.control b/contrib/isn/isn.control index 1cb5e2b2340..f816d9b0906 100644 --- a/contrib/isn/isn.control +++ b/contrib/isn/isn.control @@ -3,4 +3,4 @@ comment = 'data types for international product numbering standards' default_version = '1.2' module_pathname = '$libdir/isn' relocatable = true -trusted = true +trusted = false diff --git a/contrib/jsonb_plperl/jsonb_plperl.control b/contrib/jsonb_plperl/jsonb_plperl.control index 4acee93a2fc..b4adfad17fb 100644 --- a/contrib/jsonb_plperl/jsonb_plperl.control +++ b/contrib/jsonb_plperl/jsonb_plperl.control @@ -3,5 +3,5 @@ comment = 'transform between jsonb and plperl' default_version = '1.0' module_pathname = '$libdir/jsonb_plperl' relocatable = true -trusted = true +trusted = false requires = 'plperl' diff --git a/contrib/lo/lo.control b/contrib/lo/lo.control index f73f8b5fae5..7b3de9e111a 100644 --- a/contrib/lo/lo.control +++ b/contrib/lo/lo.control @@ -3,4 +3,4 @@ comment = 'Large Object maintenance' default_version = '1.1' module_pathname = '$libdir/lo' relocatable = true -trusted = true +trusted = false diff --git a/contrib/ltree/ltree.control b/contrib/ltree/ltree.control index c2cbeda96c7..97f697fad62 100644 --- a/contrib/ltree/ltree.control +++ b/contrib/ltree/ltree.control @@ -3,4 +3,4 @@ comment = 'data type for hierarchical tree-like structures' default_version = '1.3' module_pathname = '$libdir/ltree' relocatable = true -trusted = true +trusted = false diff --git a/contrib/pg_trgm/pg_trgm.control b/contrib/pg_trgm/pg_trgm.control index 1d6a9ddf259..ba1db412f3a 100644 --- a/contrib/pg_trgm/pg_trgm.control +++ b/contrib/pg_trgm/pg_trgm.control @@ -3,4 +3,4 @@ comment = 'text similarity measurement and index searching based on trigrams' default_version = '1.6' module_pathname = '$libdir/pg_trgm' relocatable = true -trusted = true +trusted = false diff --git a/contrib/pgcrypto/pgcrypto.control b/contrib/pgcrypto/pgcrypto.control index d2151d3bc4b..74841d0e7bd 100644 --- a/contrib/pgcrypto/pgcrypto.control +++ b/contrib/pgcrypto/pgcrypto.control @@ -3,4 +3,4 @@ comment = 'cryptographic functions' default_version = '1.3' module_pathname = '$libdir/pgcrypto' relocatable = true -trusted = true +trusted = false diff --git a/contrib/seg/seg.control b/contrib/seg/seg.control index e2c6a4750fc..0fa4236c079 100644 --- a/contrib/seg/seg.control +++ b/contrib/seg/seg.control @@ -3,4 +3,4 @@ comment = 'data type for representing line segments or floating-point intervals' default_version = '1.4' module_pathname = '$libdir/seg' relocatable = true -trusted = true +trusted = false diff --git a/contrib/tablefunc/tablefunc.control b/contrib/tablefunc/tablefunc.control index 7b25d161702..64d039977de 100644 --- a/contrib/tablefunc/tablefunc.control +++ b/contrib/tablefunc/tablefunc.control @@ -3,4 +3,4 @@ comment = 'functions that manipulate whole tables, including crosstab' default_version = '1.0' module_pathname = '$libdir/tablefunc' relocatable = true -trusted = true +trusted = false diff --git a/contrib/tcn/tcn.control b/contrib/tcn/tcn.control index 6972e1102e2..d54c22c9b49 100644 --- a/contrib/tcn/tcn.control +++ b/contrib/tcn/tcn.control @@ -3,4 +3,4 @@ comment = 'Triggered change notifications' default_version = '1.0' module_pathname = '$libdir/tcn' relocatable = true -trusted = true +trusted = false diff --git a/contrib/tsm_system_rows/tsm_system_rows.control b/contrib/tsm_system_rows/tsm_system_rows.control index b495fb126c0..ccc72d89058 100644 --- a/contrib/tsm_system_rows/tsm_system_rows.control +++ b/contrib/tsm_system_rows/tsm_system_rows.control @@ -3,4 +3,4 @@ comment = 'TABLESAMPLE method which accepts number of rows as a limit' default_version = '1.0' module_pathname = '$libdir/tsm_system_rows' relocatable = true -trusted = true +trusted = false diff --git a/contrib/tsm_system_time/tsm_system_time.control b/contrib/tsm_system_time/tsm_system_time.control index b1b9789debc..e0681787e33 100644 --- a/contrib/tsm_system_time/tsm_system_time.control +++ b/contrib/tsm_system_time/tsm_system_time.control @@ -3,4 +3,4 @@ comment = 'TABLESAMPLE method which accepts time in milliseconds as a limit' default_version = '1.0' module_pathname = '$libdir/tsm_system_time' relocatable = true -trusted = true +trusted = false diff --git a/contrib/unaccent/unaccent.control b/contrib/unaccent/unaccent.control index 649cf68a6e7..9cb1a520806 100644 --- a/contrib/unaccent/unaccent.control +++ b/contrib/unaccent/unaccent.control @@ -3,4 +3,4 @@ comment = 'text search dictionary that removes accents' default_version = '1.1' module_pathname = '$libdir/unaccent' relocatable = true -trusted = true +trusted = false diff --git a/contrib/uuid-ossp/uuid-ossp.control b/contrib/uuid-ossp/uuid-ossp.control index 142a99e4a89..8c2bd525b3d 100644 --- a/contrib/uuid-ossp/uuid-ossp.control +++ b/contrib/uuid-ossp/uuid-ossp.control @@ -3,4 +3,4 @@ comment = 'generate universally unique identifiers (UUIDs)' default_version = '1.1' module_pathname = '$libdir/uuid-ossp' relocatable = true -trusted = true +trusted = false From 8b28aa49c738fe35ea5097ff2167f93f7942c0da Mon Sep 17 00:00:00 2001 From: Andrey Borodin Date: Tue, 27 Oct 2020 14:21:21 +0500 Subject: [PATCH 18/54] Disallow cancelation of syncronous commit V1 Currently we allow to cancel awaiting of syncronous commit. Some drivers cancel query after timeout. If application will retry idempotent query, it will get confirmation of written data. This can lead to split-brain in HA scenarios. To prevent it this we add synchronous_commit_cancelation setting disalowing cancelation of syncronous replication wait Version for PostgreSQL 16 --- src/backend/access/transam/xact.c | 1 + src/backend/replication/syncrep.c | 19 +++++++++++++------ src/backend/utils/misc/guc_tables.c | 9 +++++++++ src/backend/utils/misc/postgresql.conf.sample | 5 +++++ src/include/access/xact.h | 2 ++ 5 files changed, 30 insertions(+), 6 deletions(-) diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c index 053a200d9cb..abe34c4f0e5 100644 --- a/src/backend/access/transam/xact.c +++ b/src/backend/access/transam/xact.c @@ -83,6 +83,7 @@ bool DefaultXactDeferrable = false; bool XactDeferrable; int synchronous_commit = SYNCHRONOUS_COMMIT_ON; +bool synchronous_commit_cancelation = false; /* * CheckXidAlive is a xid value pointing to a possibly ongoing (sub) diff --git a/src/backend/replication/syncrep.c b/src/backend/replication/syncrep.c index e6df5281289..18b2dcddee2 100644 --- a/src/backend/replication/syncrep.c +++ b/src/backend/replication/syncrep.c @@ -301,8 +301,8 @@ SyncRepWaitForLSN(XLogRecPtr lsn, bool commit) { ereport(WARNING, (errcode(ERRCODE_ADMIN_SHUTDOWN), - errmsg("canceling the wait for synchronous replication and terminating connection due to administrator command"), - errdetail("The transaction has already committed locally, but might not have been replicated to the standby."))); + errmsg("canceling the wait for synchronous replication and terminating connection due to administrator command"), + errdetail("The transaction has already committed locally, but might not have been replicated to the standby."))); whereToSendOutput = DestNone; SyncRepCancelWait(); break; @@ -317,11 +317,18 @@ SyncRepWaitForLSN(XLogRecPtr lsn, bool commit) if (QueryCancelPending) { QueryCancelPending = false; + if (synchronous_commit_cancelation) + { + ereport(WARNING, + (errmsg("canceling wait for synchronous replication due to user request"), + errdetail("The transaction has already committed locally, but might not have been replicated to the standby."))); + SyncRepCancelWait(); + break; + } + ereport(WARNING, - (errmsg("canceling wait for synchronous replication due to user request"), - errdetail("The transaction has already committed locally, but might not have been replicated to the standby."))); - SyncRepCancelWait(); - break; + (errmsg("canceling wait for synchronous replication due requested, but cancelation is not allowed"), + errdetail("The COMMIT record has already flushed to WAL locally and might not have been replicated to the standby. We must wait here."))); } /* diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index a79b3cd61cd..ea5b4304be7 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -1197,6 +1197,15 @@ struct config_bool ConfigureNamesBool[] = NULL, NULL, NULL }, + { + {"synchronous_commit_cancelation", PGC_USERSET, WAL_SETTINGS, + gettext_noop("Allow to cancel waiting for replication of transaction commited localy."), + NULL + }, + &synchronous_commit_cancelation, + false, NULL, NULL, NULL + }, + { {"log_checkpoints", PGC_SIGHUP, LOGGING_WHAT, gettext_noop("Logs each checkpoint."), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index e731b661c6e..92b49c758d2 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -841,6 +841,11 @@ #include_if_exists = '...' # include file only if it exists #include = '...' # include file +#------------------------------------------------------------------------------ +# MDB +#------------------------------------------------------------------------------ + +#synchronous_commit_cancelation = off #------------------------------------------------------------------------------ # CUSTOMIZED OPTIONS diff --git a/src/include/access/xact.h b/src/include/access/xact.h index 6d4439f0524..aba00d00346 100644 --- a/src/include/access/xact.h +++ b/src/include/access/xact.h @@ -81,6 +81,8 @@ typedef enum /* Synchronous commit level */ extern PGDLLIMPORT int synchronous_commit; +/* Allow cancelation of queries waiting for sync replication but commited locally */ +extern bool synchronous_commit_cancelation; /* used during logical streaming of a transaction */ extern PGDLLIMPORT TransactionId CheckXidAlive; From ae37ab9cfa873c367f0facdb95e38232f1825e94 Mon Sep 17 00:00:00 2001 From: Andrey Borodin Date: Sun, 6 Dec 2020 10:02:44 +0500 Subject: [PATCH 19/54] Extend multixact SLRU --- src/include/access/multixact.h | 4 ++++ src/include/access/subtrans.h | 3 +++ 2 files changed, 7 insertions(+) diff --git a/src/include/access/multixact.h b/src/include/access/multixact.h index 7ffd256c744..e57fa0ad9ee 100644 --- a/src/include/access/multixact.h +++ b/src/include/access/multixact.h @@ -29,6 +29,10 @@ #define MaxMultiXactOffset ((MultiXactOffset) 0xFFFFFFFF) +/* Number of SLRU buffers to use for multixact */ +#define NUM_MULTIXACTOFFSET_BUFFERS 32 +#define NUM_MULTIXACTMEMBER_BUFFERS 64 + /* * Possible multixact lock modes ("status"). The first four modes are for * tuple locks (FOR KEY SHARE, FOR SHARE, FOR NO KEY UPDATE, FOR UPDATE); the diff --git a/src/include/access/subtrans.h b/src/include/access/subtrans.h index e2213cf3fd2..6cdc7c4fe53 100644 --- a/src/include/access/subtrans.h +++ b/src/include/access/subtrans.h @@ -11,6 +11,9 @@ #ifndef SUBTRANS_H #define SUBTRANS_H +/* Number of SLRU buffers to use for subtrans */ +#define NUM_SUBTRANS_BUFFERS 64 + extern void SubTransSetParent(TransactionId xid, TransactionId parent); extern TransactionId SubTransGetParent(TransactionId xid); extern TransactionId SubTransGetTopmostTransaction(TransactionId xid); From 6636682c4838d2d8845071481b41392e6cd6e461 Mon Sep 17 00:00:00 2001 From: reshke Date: Tue, 13 Apr 2021 16:26:33 +0300 Subject: [PATCH 20/54] Mdb-admin patch and regression tests Introduces 3 functions: extern bool mdb_admin_allow_bypass_owner_checks(Oid userId, Oid ownerId); extern void check_mdb_admin_is_member_of_role(Oid member, Oid role); extern bool mdb_admin_is_member_of_role(Oid member, Oid role); To check mdb admin belongship and role-to-role ownership transfer correctness. Our mdb_admin ACL model is the following: * Any roles user or/and roles can be granted with mdb_admin * mdb_admin member can tranfser ownershup of relations, namespaces and functions to other roles, if target role in neither: superuser, pg_read_server_files, pg_write_server_files nor pg_execute_server_program. * Allow mdb_admin to create LEAKPROOF functions * mdb admin sets session replication role * [MDB-16648 + MDB-17910]: Allow mdb admin to kill specific superuser queries MDB-27288: allow mdb_admin to kill autovac + tests mdb-27228: fix expected output One more fix Fix warns --- src/backend/catalog/namespace.c | 19 +- src/backend/commands/alter.c | 11 +- src/backend/commands/functioncmds.c | 20 ++- src/backend/commands/schemacmds.c | 19 +- src/backend/commands/tablecmds.c | 16 +- src/backend/storage/ipc/signalfuncs.c | 17 +- src/backend/utils/adt/acl.c | 163 +++++++++++++++++- src/include/catalog/pg_authid.dat | 2 +- src/include/utils/acl.h | 9 + src/test/Makefile | 2 +- src/test/mdb_admin/.gitignore | 2 + src/test/mdb_admin/Makefile | 23 +++ src/test/mdb_admin/t/signals.pl | 75 ++++++++ .../regress/expected/create_function_sql.out | 4 +- src/test/regress/expected/mdb_admin.out | 100 +++++++++++ src/test/regress/expected/test_setup.out | 1 + src/test/regress/parallel_schedule | 4 + src/test/regress/sql/mdb_admin.sql | 87 ++++++++++ src/test/regress/sql/test_setup.sql | 2 + 19 files changed, 546 insertions(+), 30 deletions(-) create mode 100644 src/test/mdb_admin/.gitignore create mode 100644 src/test/mdb_admin/Makefile create mode 100644 src/test/mdb_admin/t/signals.pl create mode 100644 src/test/regress/expected/mdb_admin.out create mode 100644 src/test/regress/sql/mdb_admin.sql diff --git a/src/backend/catalog/namespace.c b/src/backend/catalog/namespace.c index 8eaa1c586c5..5a62bcb465f 100644 --- a/src/backend/catalog/namespace.c +++ b/src/backend/catalog/namespace.c @@ -3390,6 +3390,8 @@ LookupExplicitNamespace(const char *nspname, bool missing_ok) Oid namespaceId; AclResult aclresult; + HeapTuple tuple; + Oid ownerId; /* check for pg_temp alias */ if (strcmp(nspname, "pg_temp") == 0) { @@ -3407,7 +3409,22 @@ LookupExplicitNamespace(const char *nspname, bool missing_ok) if (missing_ok && !OidIsValid(namespaceId)) return InvalidOid; - aclresult = object_aclcheck(NamespaceRelationId, namespaceId, GetUserId(), ACL_USAGE); + tuple = SearchSysCache1(NAMESPACEOID, ObjectIdGetDatum(namespaceId)); + if (!HeapTupleIsValid(tuple)) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_SCHEMA), + errmsg("schema with OID %u does not exist", namespaceId))); + + ownerId = ((Form_pg_namespace) GETSTRUCT(tuple))->nspowner; + + ReleaseSysCache(tuple); + + if (!mdb_admin_allow_bypass_owner_checks(GetUserId(), ownerId)) { + aclresult = object_aclcheck(NamespaceRelationId, namespaceId, GetUserId(), ACL_USAGE); + } else { + aclresult = ACLCHECK_OK; + } + if (aclresult != ACLCHECK_OK) aclcheck_error(aclresult, OBJECT_SCHEMA, nspname); diff --git a/src/backend/commands/alter.c b/src/backend/commands/alter.c index 132283b1ef2..c1011845bb2 100644 --- a/src/backend/commands/alter.c +++ b/src/backend/commands/alter.c @@ -964,7 +964,8 @@ AlterObjectOwner_internal(Oid classId, Oid objectId, Oid new_ownerId) if (!superuser()) { /* must be owner */ - if (!has_privs_of_role(GetUserId(), old_ownerId)) + if (!has_privs_of_role(GetUserId(), old_ownerId) + && !mdb_admin_allow_bypass_owner_checks(GetUserId(), old_ownerId)) { char *objname; char namebuf[NAMEDATALEN]; @@ -985,14 +986,16 @@ AlterObjectOwner_internal(Oid classId, Oid objectId, Oid new_ownerId) get_object_type(catalogId, objectId), objname); } - /* Must be able to become new owner */ - check_can_set_role(GetUserId(), new_ownerId); + + if (!mdb_admin_is_member_of_role(GetUserId(), new_ownerId)) { + /* Must be able to become new owner */ + check_can_set_role(GetUserId(), new_ownerId); + } /* New owner must have CREATE privilege on namespace */ if (OidIsValid(namespaceId)) { AclResult aclresult; - aclresult = object_aclcheck(NamespaceRelationId, namespaceId, new_ownerId, ACL_CREATE); if (aclresult != ACLCHECK_OK) diff --git a/src/backend/commands/functioncmds.c b/src/backend/commands/functioncmds.c index d43b89d3efa..3fec7743124 100644 --- a/src/backend/commands/functioncmds.c +++ b/src/backend/commands/functioncmds.c @@ -1129,9 +1129,13 @@ CreateFunction(ParseState *pstate, CreateFunctionStmt *stmt) * by security barrier views or row-level security policies. */ if (isLeakProof && !superuser()) - ereport(ERROR, - (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), - errmsg("only superuser can define a leakproof function"))); + { + Oid role = get_role_oid("mdb_admin", true); + if (!is_member_of_role(GetUserId(), role)) + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("only superuser or mdb_admin can define a leakproof function"))); + } if (transformDefElem) { @@ -1414,9 +1418,13 @@ AlterFunction(ParseState *pstate, AlterFunctionStmt *stmt) { procForm->proleakproof = boolVal(leakproof_item->arg); if (procForm->proleakproof && !superuser()) - ereport(ERROR, - (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), - errmsg("only superuser can define a leakproof function"))); + { + Oid role = get_role_oid("mdb_admin", true); + if (!is_member_of_role(GetUserId(), role)) + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("only superuser or mdb_admin can define a leakproof function"))); + } } if (cost_item) { diff --git a/src/backend/commands/schemacmds.c b/src/backend/commands/schemacmds.c index 233f8ad1d44..b0fdbfca1e2 100644 --- a/src/backend/commands/schemacmds.c +++ b/src/backend/commands/schemacmds.c @@ -383,12 +383,16 @@ AlterSchemaOwner_internal(HeapTuple tup, Relation rel, Oid newOwnerId) AclResult aclresult; /* Otherwise, must be owner of the existing object */ - if (!object_ownercheck(NamespaceRelationId, nspForm->oid, GetUserId())) + if (!mdb_admin_allow_bypass_owner_checks(GetUserId(), nspForm->nspowner) + && !object_ownercheck(NamespaceRelationId, nspForm->oid, GetUserId())) aclcheck_error(ACLCHECK_NOT_OWNER, OBJECT_SCHEMA, NameStr(nspForm->nspname)); - /* Must be able to become new owner */ - check_can_set_role(GetUserId(), newOwnerId); + + if (!mdb_admin_is_member_of_role(GetUserId(), newOwnerId)) { + /* Must be able to become new owner */ + check_can_set_role(GetUserId(), newOwnerId); + } /* * must have create-schema rights @@ -399,8 +403,14 @@ AlterSchemaOwner_internal(HeapTuple tup, Relation rel, Oid newOwnerId) * schemas. Because superusers will always have this right, we need * no special case for them. */ - aclresult = object_aclcheck(DatabaseRelationId, MyDatabaseId, GetUserId(), + + if (mdb_admin_allow_bypass_owner_checks(GetUserId(), nspForm->nspowner)) { + aclresult = ACLCHECK_OK; + } else { + aclresult = object_aclcheck(DatabaseRelationId, MyDatabaseId, GetUserId(), ACL_CREATE); + } + if (aclresult != ACLCHECK_OK) aclcheck_error(aclresult, OBJECT_DATABASE, get_database_name(MyDatabaseId)); @@ -431,7 +441,6 @@ AlterSchemaOwner_internal(HeapTuple tup, Relation rel, Oid newOwnerId) CatalogTupleUpdate(rel, &newtuple->t_self, newtuple); heap_freetuple(newtuple); - /* Update owner dependency reference */ changeDependencyOnOwner(NamespaceRelationId, nspForm->oid, newOwnerId); diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index 3ba65a33a12..95c56754ba2 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -14599,13 +14599,18 @@ ATExecChangeOwner(Oid relationOid, Oid newOwnerId, bool recursing, LOCKMODE lock AclResult aclresult; /* Otherwise, must be owner of the existing object */ - if (!object_ownercheck(RelationRelationId, relationOid, GetUserId())) + + if (!mdb_admin_allow_bypass_owner_checks(GetUserId(), tuple_class->relowner) + && !object_ownercheck(RelationRelationId, relationOid, GetUserId())) aclcheck_error(ACLCHECK_NOT_OWNER, get_relkind_objtype(get_rel_relkind(relationOid)), RelationGetRelationName(target_rel)); - /* Must be able to become new owner */ - check_can_set_role(GetUserId(), newOwnerId); + if (!mdb_admin_is_member_of_role(GetUserId(), newOwnerId)) { + /* Must be able to become new owner */ + check_can_set_role(GetUserId(), newOwnerId); + } + /* New owner must have CREATE privilege on namespace */ aclresult = object_aclcheck(NamespaceRelationId, namespaceOid, newOwnerId, ACL_CREATE); @@ -17853,7 +17858,7 @@ RangeVarCallbackForAlterRelation(const RangeVar *rv, Oid relid, Oid oldrelid, Form_pg_class classform; AclResult aclresult; char relkind; - + tuple = SearchSysCache1(RELOID, ObjectIdGetDatum(relid)); if (!HeapTupleIsValid(tuple)) return; /* concurrently dropped */ @@ -17861,7 +17866,8 @@ RangeVarCallbackForAlterRelation(const RangeVar *rv, Oid relid, Oid oldrelid, relkind = classform->relkind; /* Must own relation. */ - if (!object_ownercheck(RelationRelationId, relid, GetUserId())) + if (!mdb_admin_allow_bypass_owner_checks(GetUserId(), classform->relowner) + && !object_ownercheck(RelationRelationId, relid, GetUserId())) aclcheck_error(ACLCHECK_NOT_OWNER, get_relkind_objtype(get_rel_relkind(relid)), rv->relname); /* No system table modifications unless explicitly allowed. */ diff --git a/src/backend/storage/ipc/signalfuncs.c b/src/backend/storage/ipc/signalfuncs.c index d398961be1f..d1a18ca4f18 100644 --- a/src/backend/storage/ipc/signalfuncs.c +++ b/src/backend/storage/ipc/signalfuncs.c @@ -20,6 +20,7 @@ #include "miscadmin.h" #include "pgstat.h" #include "postmaster/syslogger.h" +#include "postmaster/bgworker.h" #include "storage/pmsignal.h" #include "storage/proc.h" #include "storage/procarray.h" @@ -49,6 +50,9 @@ static int pg_signal_backend(int pid, int sig) { PGPROC *proc = BackendPidGetProc(pid); + LocalPgBackendStatus *local_beentry; + + local_beentry = NULL; /* * BackendPidGetProc returns NULL if the pid isn't valid; but by the time @@ -84,10 +88,15 @@ pg_signal_backend(int pid, int sig) if (!superuser()) { if (!OidIsValid(proc->roleId)) { LocalPgBackendStatus *local_beentry; - local_beentry = pgstat_get_local_beentry_by_backend_id(proc->backendId); - - if (!(local_beentry && local_beentry->backendStatus.st_backendType == B_AUTOVAC_WORKER && - has_privs_of_role(GetUserId(), ROLE_PG_SIGNAL_AUTOVACUUM))) + char * appname = NULL; + local_beentry = pgstat_get_local_beentry_by_proc_number(GetNumberFromPGProc(proc)); + if (local_beentry) { + appname = local_beentry->backendStatus.st_appname; + } + + if (!((local_beentry && local_beentry->backendStatus.st_backendType == B_AUTOVAC_WORKER && + has_privs_of_role(GetUserId(), ROLE_PG_SIGNAL_AUTOVACUUM)) + || (appname != NULL && strcmp(appname, "MDB") == 0))) return SIGNAL_BACKEND_NOSUPERUSER; } else { if (superuser_arg(proc->roleId)) diff --git a/src/backend/utils/adt/acl.c b/src/backend/utils/adt/acl.c index 6c723e0dbcc..adc5ea8a2df 100644 --- a/src/backend/utils/adt/acl.c +++ b/src/backend/utils/adt/acl.c @@ -129,6 +129,8 @@ static AclResult pg_role_aclcheck(Oid role_oid, Oid roleid, AclMode mode); static void RoleMembershipCacheCallback(Datum arg, int cacheid, uint32 hashvalue); +static bool has_privs_of_unwanted_system_role(Oid role); + /* * Test whether an identifier char can be left unquoted in ACLs. @@ -5147,6 +5149,58 @@ roles_is_member_of(Oid roleid, enum RoleRecurseType type, * * See also member_can_set_role, below. */ + +/* +* This is basically original postgresql privs-check function +*/ + +static bool +has_privs_of_role_strict(Oid member, Oid role) +{ + /* Fast path for simple case */ + if (member == role) + return true; + + /* Superusers have every privilege, so are part of every role */ + if (superuser_arg(member)) + return true; + + /* + * Find all the roles that member has the privileges of, including + * multi-level recursion, then see if target role is any one of them. + */ + return list_member_oid(roles_is_member_of(member, ROLERECURSE_PRIVS, + InvalidOid, NULL), + role); +} + +/* +* Check that role is either one of "dangerous" system role +* or has "strict" (not through mdb_admin) +* privs of this role +*/ + +static bool +has_privs_of_unwanted_system_role(Oid role) { + if (has_privs_of_role_strict(role, ROLE_PG_READ_SERVER_FILES)) { + return true; + } + if (has_privs_of_role_strict(role, ROLE_PG_WRITE_SERVER_FILES)) { + return true; + } + if (has_privs_of_role_strict(role, ROLE_PG_EXECUTE_SERVER_PROGRAM)) { + return true; + } + if (has_privs_of_role_strict(role, ROLE_PG_READ_ALL_DATA)) { + return true; + } + if (has_privs_of_role_strict(role, ROLE_PG_WRITE_ALL_DATA)) { + return true; + } + + return false; +} + bool has_privs_of_role(Oid member, Oid role) { @@ -5167,6 +5221,47 @@ has_privs_of_role(Oid member, Oid role) role); } +// -- non-upstream patch begin +/* + * Is userId allowed to bypass ownership check + * and tranfer onwership to ownerId role? + */ +bool +mdb_admin_allow_bypass_owner_checks(Oid userId, Oid ownerId) +{ + Oid mdb_admin_roleoid; + /* + * Never allow nobody to grant objects to + * superusers. + * This can result in various CVE. + * For paranoic reasons, check this even before + * membership of mdb_admin role. + */ + if (superuser_arg(ownerId)) { + return false; + } + + mdb_admin_roleoid = get_role_oid("mdb_admin", true /*if nodoby created mdb_admin role in this database*/); + /* Is userId actually member of mdb admin? */ + if (!is_member_of_role(userId, mdb_admin_roleoid)) { + /* if no, disallow. */ + return false; + } + + /* + * Now, we need to check if ownerId + * is some dangerous role to trasfer membership to. + * + * For now, we check that ownerId does not have + * priviledge to execute server program or/and + * read/write server files, or/and pg read/write all data + */ + + /* All checks passed, hope will not be hacked here (again) */ + return !has_privs_of_unwanted_system_role(ownerId); +} + +// -- non-upstream patch end /* * Can member use SET ROLE to this role? * @@ -5214,6 +5309,73 @@ check_can_set_role(Oid member, Oid role) GetUserNameFromId(role, false)))); } +// -- mdb admin patch +/* + * check_mdb_admin_is_member_of_role + * is_member_of_role with a standard permission-violation error if not in usual case + * Is case `member` in mdb_admin we check that role is neither of superuser, pg_read/write + * server files nor pg_execute_server_program or pg_read/write all data + */ +void +check_mdb_admin_is_member_of_role(Oid member, Oid role) +{ + Oid mdb_admin_roleoid; + /* fast path - if we are superuser, its ok */ + if (superuser_arg(member)) { + return; + } + + mdb_admin_roleoid = get_role_oid("mdb_admin", true /*if nodoby created mdb_admin role in this database*/); + /* Is userId actually member of mdb admin? */ + if (is_member_of_role(member, mdb_admin_roleoid)) { + + /* role is mdb admin */ + if (superuser_arg(role)) { + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("cannot transfer ownership to superuser \"%s\"", + GetUserNameFromId(role, false)))); + } + + if (has_privs_of_unwanted_system_role(role)) { + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("forbidden to transfer ownership to this system role in Cloud"))); + } + } else { + /* if no, check membership transfer in usual way. */ + + if (!is_member_of_role(member, role)) { + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("must be member of role \"%s\"", + GetUserNameFromId(role, false)))); + } + } +} + +bool mdb_admin_is_member_of_role(Oid member, Oid role) { + Oid mdb_admin_roleoid; + /* fast path - if we are superuser, its ok */ + if (superuser_arg(member)) { + return true; + } + + mdb_admin_roleoid = get_role_oid("mdb_admin", true /*if nodoby created mdb_admin role in this database*/); + /* Is userId actually member of mdb admin? */ + if (!is_member_of_role(member, mdb_admin_roleoid)) { + return false; + } + /* role is mdb admin */ + if (superuser_arg(role)) { + return false; + } + + return !has_privs_of_unwanted_system_role(role); +} + +// -- mdb admin patch + /* * Is member a member of role (directly or indirectly)? * @@ -5388,7 +5550,6 @@ select_best_grantor(Oid roleId, AclMode privileges, */ roles_list = roles_is_member_of(roleId, ROLERECURSE_PRIVS, InvalidOid, NULL); - /* initialize candidate result as default */ *grantorId = roleId; *grantOptions = ACL_NO_RIGHTS; diff --git a/src/include/catalog/pg_authid.dat b/src/include/catalog/pg_authid.dat index 63718d6a892..b986aaef91d 100644 --- a/src/include/catalog/pg_authid.dat +++ b/src/include/catalog/pg_authid.dat @@ -99,7 +99,7 @@ rolcreaterole => 'f', rolcreatedb => 'f', rolcanlogin => 'f', rolreplication => 'f', rolbypassrls => 'f', rolconnlimit => '-1', rolpassword => '_null_', rolvaliduntil => '_null_' }, -{ oid => '6312', oid_symbol => 'ROLE_PG_SIGNAL_AUTOVACUUM', +{ oid => '8004', oid_symbol => 'ROLE_PG_SIGNAL_AUTOVACUUM', rolname => 'pg_signal_autovacuum', rolsuper => 'f', rolinherit => 't', rolcreaterole => 'f', rolcreatedb => 'f', rolcanlogin => 'f', rolreplication => 'f', rolbypassrls => 'f', rolconnlimit => '-1', diff --git a/src/include/utils/acl.h b/src/include/utils/acl.h index 731d84b2a93..fa0cfa4a8db 100644 --- a/src/include/utils/acl.h +++ b/src/include/utils/acl.h @@ -215,6 +215,15 @@ extern void check_can_set_role(Oid member, Oid role); extern bool is_member_of_role(Oid member, Oid role); extern bool is_member_of_role_nosuper(Oid member, Oid role); extern bool is_admin_of_role(Oid member, Oid role); + +// -- non-upstream patch begin +extern bool mdb_admin_allow_bypass_owner_checks(Oid userId, Oid ownerId); + +extern void check_mdb_admin_is_member_of_role(Oid member, Oid role); + +extern bool mdb_admin_is_member_of_role(Oid member, Oid role); +// -- non-upstream patch end + extern Oid select_best_admin(Oid member, Oid role); extern Oid get_role_oid(const char *rolname, bool missing_ok); extern Oid get_role_oid_or_public(const char *rolname); diff --git a/src/test/Makefile b/src/test/Makefile index dbd3192874d..9563bdcdd0e 100644 --- a/src/test/Makefile +++ b/src/test/Makefile @@ -12,7 +12,7 @@ subdir = src/test top_builddir = ../.. include $(top_builddir)/src/Makefile.global -SUBDIRS = perl regress isolation modules authentication recovery subscription +SUBDIRS = perl regress isolation modules authentication recovery subscription mdb_admin ifeq ($(with_icu),yes) SUBDIRS += icu diff --git a/src/test/mdb_admin/.gitignore b/src/test/mdb_admin/.gitignore new file mode 100644 index 00000000000..871e943d50e --- /dev/null +++ b/src/test/mdb_admin/.gitignore @@ -0,0 +1,2 @@ +# Generated by test suite +/tmp_check/ diff --git a/src/test/mdb_admin/Makefile b/src/test/mdb_admin/Makefile new file mode 100644 index 00000000000..e4e82367da9 --- /dev/null +++ b/src/test/mdb_admin/Makefile @@ -0,0 +1,23 @@ +#------------------------------------------------------------------------- +# +# Makefile for src/test/mdb_admin +# +# Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group +# Portions Copyright (c) 1994, Regents of the University of California +# +# src/test/mdb_admin/Makefile +# +#------------------------------------------------------------------------- + +subdir = src/test/mdb_admin +top_builddir = ../../.. +include $(top_builddir)/src/Makefile.global + +check: + $(prove_check) + +installcheck: + $(prove_installcheck) + +clean distclean maintainer-clean: + rm -rf tmp_check diff --git a/src/test/mdb_admin/t/signals.pl b/src/test/mdb_admin/t/signals.pl new file mode 100644 index 00000000000..cb9c2e12db2 --- /dev/null +++ b/src/test/mdb_admin/t/signals.pl @@ -0,0 +1,75 @@ + +# Copyright (c) 2024-2024, MDB, Mother Russia + +# Minimal test testing streaming replication +use strict; +use warnings; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +# Initialize primary node +my $node_primary = PostgreSQL::Test::Cluster->new('primary'); +$node_primary->init(); +$node_primary->start; + +# Create some content on primary and check its presence in standby nodes +$node_primary->safe_psql('postgres', + " + CREATE DATABASE regress; + CREATE ROLE mdb_admin; + GRANT pg_signal_autovacuum TO mdb_admin; + CREATE ROLE mdb_reg_lh_1; + CREATE ROLE mdb_reg_lh_2; + GRANT pg_signal_backend TO mdb_admin; + GRANT pg_signal_backend TO mdb_reg_lh_1; + GRANT mdb_admin TO mdb_reg_lh_2; +"); + +# Create some content on primary and check its presence in standby nodes +$node_primary->safe_psql('regress', + " + CREATE TABLE tab_int(i int); + INSERT INTO tab_int SELECT * FROm generate_series(1, 1000000); + ALTER SYSTEM SET autovacuum_vacuum_cost_limit TO 1; + ALTER SYSTEM SET autovacuum_vacuum_cost_delay TO 100; + ALTER SYSTEM SET autovacuum_naptime TO 1; +"); + +$node_primary->restart; + +sleep 1; + +my $res_pid = $node_primary->safe_psql('regress', + " + SELECT pid FROM pg_stat_activity WHERE backend_type = 'autovacuum worker' and datname = 'regress';; +"); + + +print "pid is $res_pid\n"; + +ok(1); + + +my ($res_reg_lh_1, $stdout_reg_lh_1, $stderr_reg_lh_1) = $node_primary->psql('regress', + " + SET ROLE mdb_reg_lh_1; + SELECT pg_terminate_backend($res_pid); +"); + +# print ($res_reg_lh_1, $stdout_reg_lh_1, $stderr_reg_lh_1, "\n"); + +ok($res_reg_lh_1 != 0, "should fail for non-mdb_admin"); +like($stderr_reg_lh_1, qr/Only roles with the SUPERUSER attribute may terminate processes of roles with the SUPERUSER attribute./, "matches"); + +my ($res_reg_lh_2, $stdout_reg_lh_2, $stderr_reg_lh_2) = $node_primary->psql('regress', + " + SET ROLE mdb_reg_lh_2; + SELECT pg_terminate_backend($res_pid); +"); + +ok($res_reg_lh_2 == 0, "should success for mdb_admin"); + +# print ($res_reg_lh_2, $stdout_reg_lh_2, $stderr_reg_lh_2, "\n"); + +done_testing(); diff --git a/src/test/regress/expected/create_function_sql.out b/src/test/regress/expected/create_function_sql.out index 50aca5940ff..96b40fdb56d 100644 --- a/src/test/regress/expected/create_function_sql.out +++ b/src/test/regress/expected/create_function_sql.out @@ -166,10 +166,10 @@ SET SESSION AUTHORIZATION regress_unpriv_user; SET search_path TO temp_func_test, public; ALTER FUNCTION functest_E_1(int) NOT LEAKPROOF; ALTER FUNCTION functest_E_2(int) LEAKPROOF; -ERROR: only superuser can define a leakproof function +ERROR: only superuser or mdb_admin can define a leakproof function CREATE FUNCTION functest_E_3(int) RETURNS bool LANGUAGE 'sql' LEAKPROOF AS 'SELECT $1 < 200'; -- fail -ERROR: only superuser can define a leakproof function +ERROR: only superuser or mdb_admin can define a leakproof function RESET SESSION AUTHORIZATION; -- -- CALLED ON NULL INPUT | RETURNS NULL ON NULL INPUT | STRICT diff --git a/src/test/regress/expected/mdb_admin.out b/src/test/regress/expected/mdb_admin.out new file mode 100644 index 00000000000..20722750c05 --- /dev/null +++ b/src/test/regress/expected/mdb_admin.out @@ -0,0 +1,100 @@ +CREATE ROLE regress_mdb_admin_user1; +CREATE ROLE regress_mdb_admin_user2; +CREATE ROLE regress_mdb_admin_user3; +CREATE ROLE regress_superuser WITH SUPERUSER; +GRANT mdb_admin TO regress_mdb_admin_user1; +GRANT CREATE ON DATABASE regression TO regress_mdb_admin_user2; +GRANT CREATE ON DATABASE regression TO regress_mdb_admin_user3; +-- mdb admin trasfers ownership to another role +SET ROLE regress_mdb_admin_user2; +CREATE FUNCTION regress_mdb_admin_add(integer, integer) RETURNS integer + AS 'SELECT $1 + $2;' + LANGUAGE SQL + IMMUTABLE + RETURNS NULL ON NULL INPUT; +CREATE SCHEMA regress_mdb_admin_schema; +GRANT CREATE ON SCHEMA regress_mdb_admin_schema TO regress_mdb_admin_user3; +CREATE TABLE regress_mdb_admin_schema.regress_mdb_admin_table(); +CREATE TABLE regress_mdb_admin_table(); +CREATE VIEW regress_mdb_admin_view as SELECT 1; +SET ROLE regress_mdb_admin_user1; +ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO regress_mdb_admin_user3; +ALTER VIEW regress_mdb_admin_view OWNER TO regress_mdb_admin_user3; +ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO regress_mdb_admin_user3; +ALTER TABLE regress_mdb_admin_table OWNER TO regress_mdb_admin_user3; +ALTER SCHEMA regress_mdb_admin_schema OWNER TO regress_mdb_admin_user3; +-- mdb admin fails to transfer ownership to superusers and particular system roles +ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO regress_superuser; +ERROR: must be able to SET ROLE "regress_superuser" +ALTER VIEW regress_mdb_admin_view OWNER TO regress_superuser; +ERROR: must be able to SET ROLE "regress_superuser" +ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO regress_superuser; +ERROR: must be able to SET ROLE "regress_superuser" +ALTER TABLE regress_mdb_admin_table OWNER TO regress_superuser; +ERROR: must be able to SET ROLE "regress_superuser" +ALTER SCHEMA regress_mdb_admin_schema OWNER TO regress_superuser; +ERROR: must be able to SET ROLE "regress_superuser" +ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO pg_execute_server_program; +ERROR: must be able to SET ROLE "pg_execute_server_program" +ALTER VIEW regress_mdb_admin_view OWNER TO pg_execute_server_program; +ERROR: must be able to SET ROLE "pg_execute_server_program" +ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO pg_execute_server_program; +ERROR: must be able to SET ROLE "pg_execute_server_program" +ALTER TABLE regress_mdb_admin_table OWNER TO pg_execute_server_program; +ERROR: must be able to SET ROLE "pg_execute_server_program" +ALTER SCHEMA regress_mdb_admin_schema OWNER TO pg_execute_server_program; +ERROR: must be able to SET ROLE "pg_execute_server_program" +ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO pg_write_server_files; +ERROR: must be able to SET ROLE "pg_write_server_files" +ALTER VIEW regress_mdb_admin_view OWNER TO pg_write_server_files; +ERROR: must be able to SET ROLE "pg_write_server_files" +ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO pg_write_server_files; +ERROR: must be able to SET ROLE "pg_write_server_files" +ALTER TABLE regress_mdb_admin_table OWNER TO pg_write_server_files; +ERROR: must be able to SET ROLE "pg_write_server_files" +ALTER SCHEMA regress_mdb_admin_schema OWNER TO pg_write_server_files; +ERROR: must be able to SET ROLE "pg_write_server_files" +ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO pg_read_server_files; +ERROR: must be able to SET ROLE "pg_read_server_files" +ALTER VIEW regress_mdb_admin_view OWNER TO pg_read_server_files; +ERROR: must be able to SET ROLE "pg_read_server_files" +ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO pg_read_server_files; +ERROR: must be able to SET ROLE "pg_read_server_files" +ALTER TABLE regress_mdb_admin_table OWNER TO pg_read_server_files; +ERROR: must be able to SET ROLE "pg_read_server_files" +ALTER SCHEMA regress_mdb_admin_schema OWNER TO pg_read_server_files; +ERROR: must be able to SET ROLE "pg_read_server_files" +ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO pg_write_all_data; +ERROR: must be able to SET ROLE "pg_write_all_data" +ALTER VIEW regress_mdb_admin_view OWNER TO pg_write_all_data; +ERROR: must be able to SET ROLE "pg_write_all_data" +ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO pg_write_all_data; +ERROR: must be able to SET ROLE "pg_write_all_data" +ALTER TABLE regress_mdb_admin_table OWNER TO pg_write_all_data; +ERROR: must be able to SET ROLE "pg_write_all_data" +ALTER SCHEMA regress_mdb_admin_schema OWNER TO pg_write_all_data; +ERROR: must be able to SET ROLE "pg_write_all_data" +ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO pg_read_all_data; +ERROR: must be able to SET ROLE "pg_read_all_data" +ALTER VIEW regress_mdb_admin_view OWNER TO pg_read_all_data; +ERROR: must be able to SET ROLE "pg_read_all_data" +ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO pg_read_all_data; +ERROR: must be able to SET ROLE "pg_read_all_data" +ALTER TABLE regress_mdb_admin_table OWNER TO pg_read_all_data; +ERROR: must be able to SET ROLE "pg_read_all_data" +ALTER SCHEMA regress_mdb_admin_schema OWNER TO pg_read_all_data; +ERROR: must be able to SET ROLE "pg_read_all_data" +-- end tests +RESET SESSION AUTHORIZATION; +-- +REVOKE CREATE ON DATABASE regression FROM regress_mdb_admin_user2; +REVOKE CREATE ON DATABASE regression FROM regress_mdb_admin_user3; +DROP VIEW regress_mdb_admin_view; +DROP FUNCTION regress_mdb_admin_add; +DROP TABLE regress_mdb_admin_schema.regress_mdb_admin_table; +DROP TABLE regress_mdb_admin_table; +DROP SCHEMA regress_mdb_admin_schema; +DROP ROLE regress_mdb_admin_user1; +DROP ROLE regress_mdb_admin_user2; +DROP ROLE regress_mdb_admin_user3; +DROP ROLE regress_superuser; diff --git a/src/test/regress/expected/test_setup.out b/src/test/regress/expected/test_setup.out index 3d0eeec9960..882f560eb19 100644 --- a/src/test/regress/expected/test_setup.out +++ b/src/test/regress/expected/test_setup.out @@ -239,3 +239,4 @@ create function fipshash(text) returns text strict immutable parallel safe leakproof return substr(encode(sha256($1::bytea), 'hex'), 1, 32); +CREATE ROLE mdb_admin; diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule index be9f5736c44..e24513f14ff 100644 --- a/src/test/regress/parallel_schedule +++ b/src/test/regress/parallel_schedule @@ -11,6 +11,10 @@ # required setup steps test: test_setup +# mdb admin simple checks + +test: mdb_admin + # ---------- # The first group of parallel tests # ---------- diff --git a/src/test/regress/sql/mdb_admin.sql b/src/test/regress/sql/mdb_admin.sql new file mode 100644 index 00000000000..b6b048e5692 --- /dev/null +++ b/src/test/regress/sql/mdb_admin.sql @@ -0,0 +1,87 @@ +CREATE ROLE regress_mdb_admin_user1; +CREATE ROLE regress_mdb_admin_user2; +CREATE ROLE regress_mdb_admin_user3; + +CREATE ROLE regress_superuser WITH SUPERUSER; + +GRANT mdb_admin TO regress_mdb_admin_user1; +GRANT CREATE ON DATABASE regression TO regress_mdb_admin_user2; +GRANT CREATE ON DATABASE regression TO regress_mdb_admin_user3; + +-- mdb admin trasfers ownership to another role + +SET ROLE regress_mdb_admin_user2; +CREATE FUNCTION regress_mdb_admin_add(integer, integer) RETURNS integer + AS 'SELECT $1 + $2;' + LANGUAGE SQL + IMMUTABLE + RETURNS NULL ON NULL INPUT; + +CREATE SCHEMA regress_mdb_admin_schema; +GRANT CREATE ON SCHEMA regress_mdb_admin_schema TO regress_mdb_admin_user3; +CREATE TABLE regress_mdb_admin_schema.regress_mdb_admin_table(); +CREATE TABLE regress_mdb_admin_table(); +CREATE VIEW regress_mdb_admin_view as SELECT 1; +SET ROLE regress_mdb_admin_user1; + +ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO regress_mdb_admin_user3; +ALTER VIEW regress_mdb_admin_view OWNER TO regress_mdb_admin_user3; +ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO regress_mdb_admin_user3; +ALTER TABLE regress_mdb_admin_table OWNER TO regress_mdb_admin_user3; +ALTER SCHEMA regress_mdb_admin_schema OWNER TO regress_mdb_admin_user3; + + +-- mdb admin fails to transfer ownership to superusers and particular system roles + +ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO regress_superuser; +ALTER VIEW regress_mdb_admin_view OWNER TO regress_superuser; +ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO regress_superuser; +ALTER TABLE regress_mdb_admin_table OWNER TO regress_superuser; +ALTER SCHEMA regress_mdb_admin_schema OWNER TO regress_superuser; + +ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO pg_execute_server_program; +ALTER VIEW regress_mdb_admin_view OWNER TO pg_execute_server_program; +ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO pg_execute_server_program; +ALTER TABLE regress_mdb_admin_table OWNER TO pg_execute_server_program; +ALTER SCHEMA regress_mdb_admin_schema OWNER TO pg_execute_server_program; + +ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO pg_write_server_files; +ALTER VIEW regress_mdb_admin_view OWNER TO pg_write_server_files; +ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO pg_write_server_files; +ALTER TABLE regress_mdb_admin_table OWNER TO pg_write_server_files; +ALTER SCHEMA regress_mdb_admin_schema OWNER TO pg_write_server_files; + +ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO pg_read_server_files; +ALTER VIEW regress_mdb_admin_view OWNER TO pg_read_server_files; +ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO pg_read_server_files; +ALTER TABLE regress_mdb_admin_table OWNER TO pg_read_server_files; +ALTER SCHEMA regress_mdb_admin_schema OWNER TO pg_read_server_files; + +ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO pg_write_all_data; +ALTER VIEW regress_mdb_admin_view OWNER TO pg_write_all_data; +ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO pg_write_all_data; +ALTER TABLE regress_mdb_admin_table OWNER TO pg_write_all_data; +ALTER SCHEMA regress_mdb_admin_schema OWNER TO pg_write_all_data; + +ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO pg_read_all_data; +ALTER VIEW regress_mdb_admin_view OWNER TO pg_read_all_data; +ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO pg_read_all_data; +ALTER TABLE regress_mdb_admin_table OWNER TO pg_read_all_data; +ALTER SCHEMA regress_mdb_admin_schema OWNER TO pg_read_all_data; + +-- end tests + +RESET SESSION AUTHORIZATION; +-- +REVOKE CREATE ON DATABASE regression FROM regress_mdb_admin_user2; +REVOKE CREATE ON DATABASE regression FROM regress_mdb_admin_user3; + +DROP VIEW regress_mdb_admin_view; +DROP FUNCTION regress_mdb_admin_add; +DROP TABLE regress_mdb_admin_schema.regress_mdb_admin_table; +DROP TABLE regress_mdb_admin_table; +DROP SCHEMA regress_mdb_admin_schema; +DROP ROLE regress_mdb_admin_user1; +DROP ROLE regress_mdb_admin_user2; +DROP ROLE regress_mdb_admin_user3; +DROP ROLE regress_superuser; diff --git a/src/test/regress/sql/test_setup.sql b/src/test/regress/sql/test_setup.sql index 06b0e2121f8..2bdcd9f2161 100644 --- a/src/test/regress/sql/test_setup.sql +++ b/src/test/regress/sql/test_setup.sql @@ -294,3 +294,5 @@ create function fipshash(text) returns text strict immutable parallel safe leakproof return substr(encode(sha256($1::bytea), 'hex'), 1, 32); + +CREATE ROLE mdb_admin; From c79e7018fb53f659d9da347ee94f4614e80446a9 Mon Sep 17 00:00:00 2001 From: Andrey Borodin Date: Sun, 6 Dec 2020 10:01:59 +0500 Subject: [PATCH 21/54] MDB replication role patch MDB replication role regression tests Patch allows user with mdb_replication role to use pg_create_logical_replication_slot, pg_replication_slot_advance, pg_drop_replication_slot functions to manage logical replication slots. Also, users with mdb_admin (which is memner of pg_create_subscription) can create subscribptions. Slot names starting with MDB.* are forbidden. Add run as owner tap tests More test cases in mdb_102 Fix test Fix tests once again Never check for superuser in walsender --- .../test_decoding/expected/permissions.out | 94 ++++- contrib/test_decoding/sql/permissions.sql | 29 ++ src/backend/commands/subscriptioncmds.c | 10 +- src/backend/postmaster/syslogger.c | 2 + .../replication/logical/logicalfuncs.c | 7 +- src/backend/replication/logical/relation.c | 7 + src/backend/replication/logical/tablesync.c | 7 + src/backend/replication/logical/worker.c | 31 +- src/backend/replication/slot.c | 38 ++ src/backend/replication/slotfuncs.c | 14 +- src/backend/replication/walsender.c | 28 ++ src/backend/utils/init/postinit.c | 14 +- src/include/replication/slot.h | 27 ++ src/include/replication/walsender.h | 4 + .../t/101_logical_decoding_mdb_replication.pl | 159 +++++++++ src/test/regress/expected/mdb_replication.out | 35 ++ src/test/regress/expected/test_setup.out | 3 + src/test/regress/parallel_schedule | 4 + src/test/regress/sql/mdb_replication.sql | 41 +++ src/test/regress/sql/test_setup.sql | 4 + .../t/101_mdb_rep_changes_nonsuperuser.pl | 336 ++++++++++++++++++ .../t/102_mdb_check_permissions.pl | 202 +++++++++++ 22 files changed, 1080 insertions(+), 16 deletions(-) create mode 100644 src/test/recovery/t/101_logical_decoding_mdb_replication.pl create mode 100644 src/test/regress/expected/mdb_replication.out create mode 100644 src/test/regress/sql/mdb_replication.sql create mode 100644 src/test/subscription/t/101_mdb_rep_changes_nonsuperuser.pl create mode 100644 src/test/subscription/t/102_mdb_check_permissions.pl diff --git a/contrib/test_decoding/expected/permissions.out b/contrib/test_decoding/expected/permissions.out index 8d100646ce6..712679e3739 100644 --- a/contrib/test_decoding/expected/permissions.out +++ b/contrib/test_decoding/expected/permissions.out @@ -2,6 +2,9 @@ SET synchronous_commit = on; -- setup CREATE ROLE regress_lr_normal; +CREATE ROLE mdb_replication; +CREATE ROLE regress_lr_mdb_replication; +GRANT mdb_replication TO regress_lr_mdb_replication; CREATE ROLE regress_lr_superuser SUPERUSER; CREATE ROLE regress_lr_replication REPLICATION; CREATE TABLE lr_test(data text); @@ -28,6 +31,24 @@ SELECT pg_drop_replication_slot('regression_slot'); (1 row) +-- superuser can control slot starts from mdb +SELECT 'init' FROM pg_create_logical_replication_slot('mdb_regression_slot', 'test_decoding'); + ?column? +---------- + init +(1 row) + +SELECT data FROM pg_logical_slot_get_changes('mdb_regression_slot', NULL, NULL, 'include-xids', '0', 'skip-empty-xacts', '1'); + data +------ +(0 rows) + +SELECT pg_drop_replication_slot('mdb_regression_slot'); + pg_drop_replication_slot +-------------------------- + +(1 row) + RESET ROLE; -- replication user can control replication SET ROLE regress_lr_replication; @@ -50,20 +71,85 @@ SELECT pg_drop_replication_slot('regression_slot'); (1 row) +-- replication user can control slot starts from mdb +SELECT 'init' FROM pg_create_logical_replication_slot('mdb_regression_slot', 'test_decoding'); + ?column? +---------- + init +(1 row) + +SELECT data FROM pg_logical_slot_get_changes('mdb_regression_slot', NULL, NULL, 'include-xids', '0', 'skip-empty-xacts', '1'); + data +------ +(0 rows) + +SELECT pg_drop_replication_slot('mdb_regression_slot'); + pg_drop_replication_slot +-------------------------- + +(1 row) + +SELECT 'init' FROM pg_create_logical_replication_slot('mdb_regression_slot', 'test_decoding'); + ?column? +---------- + init +(1 row) + +RESET ROLE; +-- mdb_replication user can control replication +SET ROLE regress_lr_mdb_replication; +SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'test_decoding'); + ?column? +---------- + init +(1 row) + +INSERT INTO lr_test VALUES('lr_mdb_replication_init'); +ERROR: permission denied for table lr_test +SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'include-xids', '0', 'skip-empty-xacts', '1'); + data +------ +(0 rows) + +SELECT pg_drop_replication_slot('regression_slot'); + pg_drop_replication_slot +-------------------------- + +(1 row) + +-- mdb_replication user can't control slot starts from mdb +SELECT 'init' FROM pg_create_logical_replication_slot('mdb_regression_slot2', 'test_decoding'); +ERROR: slot name "mdb_regression_slot2" is reserved +DETAIL: Slot names starting with "mdb" are reserved. +SELECT data FROM pg_logical_slot_get_changes('mdb_regression_slot', NULL, NULL, 'include-xids', '0', 'skip-empty-xacts', '1'); +ERROR: slot name "mdb_regression_slot" is reserved +DETAIL: Slot names starting with "mdb" are reserved. +SELECT pg_drop_replication_slot('mdb_regression_slot'); +ERROR: slot name "mdb_regression_slot" is reserved +DETAIL: Slot names starting with "mdb" are reserved. +RESET ROLE; +-- cleanup +SET ROLE regress_lr_superuser; +SELECT pg_drop_replication_slot('mdb_regression_slot'); + pg_drop_replication_slot +-------------------------- + +(1 row) + RESET ROLE; -- plain user *can't* can control replication SET ROLE regress_lr_normal; SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'test_decoding'); ERROR: permission denied to use replication slots -DETAIL: Only roles with the REPLICATION attribute may use replication slots. +DETAIL: must be superuser, replication role or mdb_replication to use replication slots INSERT INTO lr_test VALUES('lr_superuser_init'); ERROR: permission denied for table lr_test SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'include-xids', '0', 'skip-empty-xacts', '1'); ERROR: permission denied to use replication slots -DETAIL: Only roles with the REPLICATION attribute may use replication slots. +DETAIL: must be superuser, replication role or mdb_replication to use replication slots SELECT pg_drop_replication_slot('regression_slot'); ERROR: permission denied to use replication slots -DETAIL: Only roles with the REPLICATION attribute may use replication slots. +DETAIL: must be superuser, replication role or mdb_replication to use replication slots SELECT pg_sync_replication_slots(); ERROR: permission denied to use replication slots DETAIL: Only roles with the REPLICATION attribute may use replication slots. @@ -97,7 +183,7 @@ RESET ROLE; SET ROLE regress_lr_normal; SELECT pg_drop_replication_slot('regression_slot'); ERROR: permission denied to use replication slots -DETAIL: Only roles with the REPLICATION attribute may use replication slots. +DETAIL: must be superuser, replication role or mdb_replication to use replication slots RESET ROLE; -- all users can see existing slots SET ROLE regress_lr_superuser; diff --git a/contrib/test_decoding/sql/permissions.sql b/contrib/test_decoding/sql/permissions.sql index 94db936aee2..80c91eba619 100644 --- a/contrib/test_decoding/sql/permissions.sql +++ b/contrib/test_decoding/sql/permissions.sql @@ -3,6 +3,9 @@ SET synchronous_commit = on; -- setup CREATE ROLE regress_lr_normal; +CREATE ROLE mdb_replication; +CREATE ROLE regress_lr_mdb_replication; +GRANT mdb_replication TO regress_lr_mdb_replication; CREATE ROLE regress_lr_superuser SUPERUSER; CREATE ROLE regress_lr_replication REPLICATION; CREATE TABLE lr_test(data text); @@ -13,6 +16,10 @@ SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'test_d INSERT INTO lr_test VALUES('lr_superuser_init'); SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'include-xids', '0', 'skip-empty-xacts', '1'); SELECT pg_drop_replication_slot('regression_slot'); +-- superuser can control slot starts from mdb +SELECT 'init' FROM pg_create_logical_replication_slot('mdb_regression_slot', 'test_decoding'); +SELECT data FROM pg_logical_slot_get_changes('mdb_regression_slot', NULL, NULL, 'include-xids', '0', 'skip-empty-xacts', '1'); +SELECT pg_drop_replication_slot('mdb_regression_slot'); RESET ROLE; -- replication user can control replication @@ -21,6 +28,28 @@ SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'test_d INSERT INTO lr_test VALUES('lr_superuser_init'); SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'include-xids', '0', 'skip-empty-xacts', '1'); SELECT pg_drop_replication_slot('regression_slot'); +-- replication user can control slot starts from mdb +SELECT 'init' FROM pg_create_logical_replication_slot('mdb_regression_slot', 'test_decoding'); +SELECT data FROM pg_logical_slot_get_changes('mdb_regression_slot', NULL, NULL, 'include-xids', '0', 'skip-empty-xacts', '1'); +SELECT pg_drop_replication_slot('mdb_regression_slot'); +SELECT 'init' FROM pg_create_logical_replication_slot('mdb_regression_slot', 'test_decoding'); +RESET ROLE; + +-- mdb_replication user can control replication +SET ROLE regress_lr_mdb_replication; +SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'test_decoding'); +INSERT INTO lr_test VALUES('lr_mdb_replication_init'); +SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'include-xids', '0', 'skip-empty-xacts', '1'); +SELECT pg_drop_replication_slot('regression_slot'); +-- mdb_replication user can't control slot starts from mdb +SELECT 'init' FROM pg_create_logical_replication_slot('mdb_regression_slot2', 'test_decoding'); +SELECT data FROM pg_logical_slot_get_changes('mdb_regression_slot', NULL, NULL, 'include-xids', '0', 'skip-empty-xacts', '1'); +SELECT pg_drop_replication_slot('mdb_regression_slot'); +RESET ROLE; + +-- cleanup +SET ROLE regress_lr_superuser; +SELECT pg_drop_replication_slot('mdb_regression_slot'); RESET ROLE; -- plain user *can't* can control replication diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c index ef051969c24..c06154973f7 100644 --- a/src/backend/commands/subscriptioncmds.c +++ b/src/backend/commands/subscriptioncmds.c @@ -592,7 +592,7 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt, bits32 supported_opts; SubOpts opts = {0}; AclResult aclresult; - + /* * Parse and check options. * @@ -620,6 +620,7 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt, * attempts to access arbitrary network destinations, so require the user * to have been specifically authorized to create subscriptions. */ + /* MDB: mdb_admin need to be granted with pg_create_subscription role */ if (!has_privs_of_role(owner, ROLE_PG_CREATE_SUBSCRIPTION)) ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), @@ -1920,8 +1921,11 @@ AlterSubscriptionOwner_internal(Relation rel, HeapTuple tup, Oid newOwnerId) errmsg("password_required=false is superuser-only"), errhint("Subscriptions with the password_required option set to false may only be created or modified by the superuser."))); - /* Must be able to become new owner */ - check_can_set_role(GetUserId(), newOwnerId); + /* if we are mdb_admin, check that we alter ownership to unprivileged role */ + if (!mdb_admin_allow_bypass_owner_checks(GetUserId(), newOwnerId)) { + /* else we must be able to become new owner */ + check_can_set_role(GetUserId(), newOwnerId); + } /* * current owner must have CREATE on database diff --git a/src/backend/postmaster/syslogger.c b/src/backend/postmaster/syslogger.c index 437947dbb9d..2d01e93e731 100644 --- a/src/backend/postmaster/syslogger.c +++ b/src/backend/postmaster/syslogger.c @@ -344,6 +344,8 @@ SysLoggerMain(char *startup_data, size_t startup_data_len) #ifndef WIN32 AddWaitEventToSet(wes, WL_SOCKET_READABLE, syslogPipe[0], NULL, NULL); #endif + fclose(stderr); + fclose(stdout); /* main worker loop */ for (;;) diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c index b4dd5cce75b..12c71fcc0c1 100644 --- a/src/backend/replication/logical/logicalfuncs.c +++ b/src/backend/replication/logical/logicalfuncs.c @@ -28,6 +28,8 @@ #include "replication/decode.h" #include "replication/logical.h" #include "replication/message.h" +#include "storage/fd.h" +#include "utils/acl.h" #include "utils/array.h" #include "utils/builtins.h" #include "utils/inval.h" @@ -113,8 +115,7 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin List *options = NIL; DecodingOutputState *p; - CheckSlotPermissions(); - + CheckMDBReplSlotPermissions(); CheckLogicalDecodingRequirements(); if (PG_ARGISNULL(0)) @@ -123,6 +124,8 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin errmsg("slot name must not be null"))); name = PG_GETARG_NAME(0); + CheckMDBReservedName(NameStr(*name)); + if (PG_ARGISNULL(1)) upto_lsn = InvalidXLogRecPtr; else diff --git a/src/backend/replication/logical/relation.c b/src/backend/replication/logical/relation.c index f139e7b01e9..d9de84b23e0 100644 --- a/src/backend/replication/logical/relation.c +++ b/src/backend/replication/logical/relation.c @@ -22,6 +22,7 @@ #endif #include "access/genam.h" #include "access/table.h" +#include "catalog/catalog.h" #include "catalog/namespace.h" #include "catalog/pg_subscription_rel.h" #include "executor/executor.h" @@ -400,6 +401,12 @@ logicalrep_rel_open(LogicalRepRelId remoteid, LOCKMODE lockmode) entry->localrel = table_open(relid, NoLock); entry->localreloid = relid; + /* Don't allow catalog access */ + if (IsSystemClass(relid, entry->localrel->rd_rel)) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("logical replication target relation \"%s.%s\" is a system class", + remoterel->nspname, remoterel->relname))); /* Check for supported relkind. */ CheckSubscriptionRelkind(entry->localrel->rd_rel->relkind, remoterel->nspname, remoterel->relname); diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c index bd5b8755b4a..db62ae4a95e 100644 --- a/src/backend/replication/logical/tablesync.c +++ b/src/backend/replication/logical/tablesync.c @@ -122,6 +122,7 @@ #include "utils/snapmgr.h" #include "utils/syscache.h" #include "utils/usercontext.h" +#include "utils/acl.h" typedef enum { @@ -1149,6 +1150,7 @@ copy_table(Relation rel) List *attnamelist; ParseState *pstate; List *options = NIL; + AclResult aclresult; /* Get the publisher relation info. */ fetch_remote_table_info(get_namespace_name(RelationGetNamespace(rel)), @@ -1161,6 +1163,11 @@ copy_table(Relation rel) relmapentry = logicalrep_rel_open(lrel.remoteid, NoLock); Assert(rel == relmapentry->localrel); + /* Check permission on table. */ + aclresult = pg_class_aclcheck(RelationGetRelid(rel), GetUserId(), ACL_INSERT); + if (aclresult != ACLCHECK_OK) + aclcheck_error(aclresult, get_relkind_objtype(rel->rd_rel->relkind), RelationGetRelationName(rel)); + /* Start copy on the publisher. */ initStringInfo(&cmd); diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c index 313d3a82660..764e53948b3 100644 --- a/src/backend/replication/logical/worker.c +++ b/src/backend/replication/logical/worker.c @@ -191,6 +191,7 @@ #include "utils/snapmgr.h" #include "utils/syscache.h" #include "utils/usercontext.h" +#include "utils/acl.h" #define NAPTIME_PER_CYCLE 1000 /* max sleep time between cycles (1s) */ @@ -2381,6 +2382,7 @@ apply_handle_insert(StringInfo s) TupleTableSlot *remoteslot; MemoryContext oldctx; bool run_as_owner; + AclResult aclresult; /* * Quick return if we are skipping data modification changes or handling @@ -2415,6 +2417,11 @@ apply_handle_insert(StringInfo s) /* Set relation for error callback */ apply_error_callback_arg.rel = rel; + aclresult = pg_class_aclcheck(RelationGetRelid(rel->localrel), GetUserId(), + ACL_INSERT); + if (aclresult != ACLCHECK_OK) + aclcheck_error(aclresult, get_relkind_objtype(rel->localrel->rd_rel->relkind), + RelationGetRelationName(rel->localrel)); /* Initialize the executor state. */ edata = create_edata_for_relation(rel); @@ -2537,6 +2544,7 @@ apply_handle_update(StringInfo s) RTEPermissionInfo *target_perminfo; MemoryContext oldctx; bool run_as_owner; + AclResult aclresult; /* * Quick return if we are skipping data modification changes or handling @@ -2576,6 +2584,14 @@ apply_handle_update(StringInfo s) if (!run_as_owner) SwitchToUntrustedUser(rel->localrel->rd_rel->relowner, &ucxt); + /* MDB check acl on relation */ + aclresult = pg_class_aclcheck(RelationGetRelid(rel->localrel), GetUserId(), + ACL_UPDATE); + if (aclresult != ACLCHECK_OK) + aclcheck_error(aclresult, get_relkind_objtype(rel->localrel->rd_rel->relkind), + RelationGetRelationName(rel->localrel)); + + /* Initialize the executor state. */ edata = create_edata_for_relation(rel); estate = edata->estate; @@ -2718,6 +2734,7 @@ apply_handle_delete(StringInfo s) TupleTableSlot *remoteslot; MemoryContext oldctx; bool run_as_owner; + AclResult aclresult; /* * Quick return if we are skipping data modification changes or handling @@ -2744,7 +2761,6 @@ apply_handle_delete(StringInfo s) /* Set relation for error callback */ apply_error_callback_arg.rel = rel; - /* Check if we can do the delete. */ check_relation_updatable(rel); @@ -2756,6 +2772,12 @@ apply_handle_delete(StringInfo s) if (!run_as_owner) SwitchToUntrustedUser(rel->localrel->rd_rel->relowner, &ucxt); + aclresult = pg_class_aclcheck(RelationGetRelid(rel->localrel), GetUserId(), + ACL_DELETE); + if (aclresult != ACLCHECK_OK) + aclcheck_error(aclresult, get_relkind_objtype(rel->localrel->rd_rel->relkind), + RelationGetRelationName(rel->localrel)); + /* Initialize the executor state. */ edata = create_edata_for_relation(rel); estate = edata->estate; @@ -3166,6 +3188,7 @@ apply_handle_truncate(StringInfo s) List *relids_logged = NIL; ListCell *lc; LOCKMODE lockmode = AccessExclusiveLock; + AclResult aclresult; /* * Quick return if we are skipping data modification changes or handling @@ -3195,6 +3218,12 @@ apply_handle_truncate(StringInfo s) continue; } + aclresult = pg_class_aclcheck(RelationGetRelid(rel->localrel), GetUserId(), + ACL_TRUNCATE); + if (aclresult != ACLCHECK_OK) + aclcheck_error(aclresult, get_relkind_objtype(rel->localrel->rd_rel->relkind), + RelationGetRelationName(rel->localrel)); + remote_rels = lappend(remote_rels, rel); TargetPrivilegesCheck(rel->localrel, ACL_TRUNCATE); rels = lappend(rels, rel->localrel); diff --git a/src/backend/replication/slot.c b/src/backend/replication/slot.c index 4e54a8d0f14..bee3735fa23 100644 --- a/src/backend/replication/slot.c +++ b/src/backend/replication/slot.c @@ -57,6 +57,7 @@ #include "utils/builtins.h" #include "utils/guc_hooks.h" #include "utils/varlena.h" +#include "utils/acl.h" /* * Replication slot on-disk data structure. @@ -1449,6 +1450,43 @@ CheckSlotPermissions(void) "REPLICATION"))); } + + +/* + * Check whether the user has privilege to use replication slots. + */ +void +CheckRoleMDBReplSlotPermissions(bool role_has_rolreplication, bool is_member_of_mdb_replication) +{ + /* mdb_replication can do it */ + if (is_member_of_mdb_replication) { + return; + } + + if (!role_has_rolreplication) + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("permission denied to use replication slots"), + errdetail("must be superuser, replication role or mdb_replication to use replication slots"))); +} + + +void +CheckRoleUseMDBReservedName(const char *name, bool role_has_rolreplication) +{ + /* ugly coding for speed (taken from IsReservedName) */ + if (name[0] == 'm' && + name[1] == 'd' && + name[2] == 'b' && + !role_has_rolreplication) + { + ereport(ERROR, + (errcode(ERRCODE_RESERVED_NAME), + errmsg("slot name \"%s\" is reserved", name), + errdetail("Slot names starting with \"mdb\" are reserved."))); + } +} + /* * Reserve WAL for the currently active slot. * diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c index 01e98bc1cef..6d4e10faa5d 100644 --- a/src/backend/replication/slotfuncs.c +++ b/src/backend/replication/slotfuncs.c @@ -27,6 +27,7 @@ #include "utils/inval.h" #include "utils/pg_lsn.h" #include "utils/resowner.h" +#include "utils/acl.h" /* * Helper function for creating a new physical replication slot with @@ -81,6 +82,7 @@ pg_create_physical_replication_slot(PG_FUNCTION_ARGS) elog(ERROR, "return type must be a row type"); CheckSlotPermissions(); + CheckMDBReservedName(NameStr(*name)); CheckSlotRequirements(); @@ -186,7 +188,8 @@ pg_create_logical_replication_slot(PG_FUNCTION_ARGS) if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) elog(ERROR, "return type must be a row type"); - CheckSlotPermissions(); + CheckMDBReplSlotPermissions(); + CheckMDBReservedName(NameStr(*name)); CheckLogicalDecodingRequirements(); @@ -223,7 +226,9 @@ pg_drop_replication_slot(PG_FUNCTION_ARGS) { Name name = PG_GETARG_NAME(0); - CheckSlotPermissions(); + /* mdb replication allowed */ + CheckMDBReplSlotPermissions(); + CheckMDBReservedName(NameStr(*name)); CheckSlotRequirements(); @@ -519,7 +524,8 @@ pg_replication_slot_advance(PG_FUNCTION_ARGS) Assert(!MyReplicationSlot); - CheckSlotPermissions(); + CheckMDBReplSlotPermissions(); + CheckMDBReservedName(NameStr(*slotname)); if (XLogRecPtrIsInvalid(moveto)) ereport(ERROR, @@ -619,6 +625,8 @@ copy_replication_slot(FunctionCallInfo fcinfo, bool logical_slot) elog(ERROR, "return type must be a row type"); CheckSlotPermissions(); + CheckMDBReservedName(NameStr(*src_name)); + CheckMDBReservedName(NameStr(*dst_name)); if (logical_slot) CheckLogicalDecodingRequirements(); diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 5cf988122db..dceb696b86f 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -117,6 +117,13 @@ WalSnd *MyWalSnd = NULL; /* Global state */ bool am_walsender = false; /* Am I a walsender process? */ + +/* These variables defined in InitPostgres and used in walsender logic for priv +* check without CatCache search. +*/ +bool role_has_rolreplication = false; /* has replication privelege */ +bool member_of_mdb_replication = false; /* member of mdb replication role */ + bool am_cascading_walsender = false; /* Am I cascading WAL to another * standby? */ bool am_db_walsender = false; /* Connected to a database? */ @@ -221,6 +228,20 @@ typedef struct /* The size of our buffer of time samples. */ #define LAG_TRACKER_BUFFER_SIZE 8192 +static void +check_permissions(void) +{ + /* superuser can do it */ + if (superuser()) { + return; + } + /* else should have REPLICATION role option */ + if (!role_has_rolreplication) + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + (errmsg("must be superuser or replication role to use replication slots")))); +} + /* A mechanism for tracking replication lag. */ typedef struct { @@ -1237,6 +1258,7 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd) if (cmd->kind == REPLICATION_KIND_PHYSICAL) { + check_permissions(); ReplicationSlotCreate(cmd->slotname, false, cmd->temporary ? RS_TEMPORARY : RS_PERSISTENT, false, false, false); @@ -1260,6 +1282,8 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd) Assert(cmd->kind == REPLICATION_KIND_LOGICAL); CheckLogicalDecodingRequirements(); + CheckRoleMDBReplSlotPermissions(role_has_rolreplication, member_of_mdb_replication); + CheckRoleUseMDBReservedName(cmd->slotname, role_has_rolreplication); /* * Initially create persistent slot as ephemeral - that allows us to @@ -1422,6 +1446,7 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd) static void DropReplicationSlot(DropReplicationSlotCmd *cmd) { + CheckRoleUseMDBReservedName(cmd->slotname, role_has_rolreplication); ReplicationSlotDrop(cmd->slotname, !cmd->wait); } @@ -1472,6 +1497,8 @@ StartLogicalReplication(StartReplicationCmd *cmd) StringInfoData buf; QueryCompletion qc; + CheckRoleUseMDBReservedName(cmd->slotname, role_has_rolreplication); + /* make sure that our requirements are still fulfilled */ CheckLogicalDecodingRequirements(); @@ -2138,6 +2165,7 @@ exec_replication_command(const char *cmd_string) break; case T_BaseBackupCmd: + check_permissions(); cmdtag = "BASE_BACKUP"; set_ps_display(cmdtag); PreventInTransactionBlock(true, cmdtag); diff --git a/src/backend/utils/init/postinit.c b/src/backend/utils/init/postinit.c index 2ed7c7c02db..fd4a3799c20 100644 --- a/src/backend/utils/init/postinit.c +++ b/src/backend/utils/init/postinit.c @@ -744,7 +744,7 @@ InitPostgres(const char *in_dbname, Oid dboid, bool am_superuser; char *fullpath; char dbname[NAMEDATALEN]; - int nfree = 0; + int nfree; elog(DEBUG3, "InitPostgres"); @@ -973,8 +973,16 @@ InitPostgres(const char *in_dbname, Oid dboid, if (am_walsender) { Assert(!bootstrap); - - if (!has_rolreplication(GetUserId())) + /* define this variable for later use in permission checks function. */ + /* we cannot use has_rolreplication directly because catcache search is prohibited + * in no active tx state. + */ + role_has_rolreplication = has_rolreplication(GetUserId()); + member_of_mdb_replication = is_member_of_role(GetUserId(), get_role_oid("mdb_replication", true)); + + /* has_rolreplication returns true in case of superuser_arg(role) */ + /* should have REPLICATION role or be a member of mdb_replication to start walsender */ + if (!role_has_rolreplication && !member_of_mdb_replication) ereport(FATAL, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("permission denied to start WAL sender"), diff --git a/src/include/replication/slot.h b/src/include/replication/slot.h index 43594bb9bef..21159782f36 100644 --- a/src/include/replication/slot.h +++ b/src/include/replication/slot.h @@ -9,6 +9,8 @@ #ifndef SLOT_H #define SLOT_H +#include "utils/acl.h" +#include "miscadmin.h" #include "access/xlog.h" #include "access/xlogreader.h" #include "storage/condition_variable.h" @@ -288,4 +290,29 @@ extern bool SlotExistsInSyncStandbySlots(const char *slot_name); extern bool StandbySlotsHaveCaughtup(XLogRecPtr wait_for_lsn, int elevel); extern void WaitForStandbyConfirmation(XLogRecPtr wait_for_lsn); +/* +* Base function for CheckMDBReplSlotPermissions, but does not +* perform cat cache search (for no-transaction state case), +* using pre-defined bool variables. +*/ +extern void CheckRoleMDBReplSlotPermissions(bool role_has_rolreplication, bool is_member_of_mdb_replication); + +/* +* Same as CheckMDBReservedName, but does not +* perform cat cache search (for no-transaction state case), +* using pre-defined bool variables, defined +* in InitPostrges. +*/ +extern void CheckRoleUseMDBReservedName(const char* name, bool role_has_rolreplication); + +inline void CheckMDBReservedName(const char* name) { + CheckRoleUseMDBReservedName(name, has_rolreplication(GetUserId())); +} + +inline void CheckMDBReplSlotPermissions(void) { + Oid role; + role = get_role_oid("mdb_replication", /* missing ok*/ true); + return CheckRoleMDBReplSlotPermissions(has_rolreplication(GetUserId()), is_member_of_role(GetUserId(), role)); +} + #endif /* SLOT_H */ diff --git a/src/include/replication/walsender.h b/src/include/replication/walsender.h index f2d8297f016..f3be3e00716 100644 --- a/src/include/replication/walsender.h +++ b/src/include/replication/walsender.h @@ -30,6 +30,10 @@ extern PGDLLIMPORT bool am_cascading_walsender; extern PGDLLIMPORT bool am_db_walsender; extern PGDLLIMPORT bool wake_wal_senders; +/* for mdb permission checks */ +extern PGDLLIMPORT bool role_has_rolreplication; +extern PGDLLIMPORT bool member_of_mdb_replication; + /* user-settable parameters */ extern PGDLLIMPORT int max_wal_senders; extern PGDLLIMPORT int wal_sender_timeout; diff --git a/src/test/recovery/t/101_logical_decoding_mdb_replication.pl b/src/test/recovery/t/101_logical_decoding_mdb_replication.pl new file mode 100644 index 00000000000..106e2484db7 --- /dev/null +++ b/src/test/recovery/t/101_logical_decoding_mdb_replication.pl @@ -0,0 +1,159 @@ +# Testing of logical decoding using SQL interface and/or pg_recvlogical +# +# Most logical decoding tests are in contrib/test_decoding. This module +# is for work that doesn't fit well there, like where server restarts +# are required. +use strict; +use warnings; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More tests => 10; +use Config; + + +# Initialize master node +my $node_master = PostgreSQL::Test::Cluster->new('master'); +$node_master->init(allows_streaming => 1); +$node_master->append_conf( + 'postgresql.conf', qq( +wal_level = logical +)); +$node_master->start; +my $backup_name = 'master_backup'; + +$node_master->safe_psql('postgres', + "SET password_encryption='md5'; CREATE ROLE normal LOGIN PASSWORD 'pass';"); +$node_master->safe_psql('postgres', + "GRANT CREATE ON DATABASE postgres TO normal;"); +$node_master->safe_psql('postgres', + "GRANT ALL ON SCHEMA public TO normal;"); +$node_master->safe_psql('postgres', + "CREATE ROLE mdb_replication NOLOGIN;"); +$node_master->safe_psql('postgres', + "GRANT mdb_replication TO normal;"); + +$node_master->safe_psql('postgres', + qq[CREATE TABLE decoding_test(x integer, y text);], extra_params => [ '-U', 'normal' ]); + +$node_master->safe_psql('postgres', + qq[SELECT pg_create_logical_replication_slot('test_slot', 'test_decoding');], + extra_params => [ '-U', 'normal' ]); + +$node_master->safe_psql('postgres', + qq[INSERT INTO decoding_test(x,y) SELECT s, s::text FROM generate_series(1,10) s;], + extra_params => [ '-U', 'normal' ] +); + +# Basic decoding works +my ($result) = $node_master->safe_psql('postgres', + qq[SELECT pg_logical_slot_get_changes('test_slot', NULL, NULL);], + extra_params => [ '-U', 'normal' ]); +is(scalar(my @foobar = split /^/m, $result), + 12, 'Decoding produced 12 rows inc BEGIN/COMMIT'); + +# If we immediately crash the server we might lose the progress we just made +# and replay the same changes again. But a clean shutdown should never repeat +# the same changes when we use the SQL decoding interface. +$node_master->restart; + +# There are no new writes, so the result should be empty. +$result = $node_master->safe_psql('postgres', + qq[SELECT pg_logical_slot_get_changes('test_slot', NULL, NULL);], + extra_params => [ '-U', 'normal' ]); +chomp($result); +is($result, '', 'Decoding after fast restart repeats no rows'); + +# Insert some rows and verify that we get the same results from pg_recvlogical +# and the SQL interface. +$node_master->safe_psql('postgres', + qq[INSERT INTO decoding_test(x,y) SELECT s, s::text FROM generate_series(1,4) s;], + extra_params => [ '-U', 'normal' ] +); + +my $expected = q{BEGIN +table public.decoding_test: INSERT: x[integer]:1 y[text]:'1' +table public.decoding_test: INSERT: x[integer]:2 y[text]:'2' +table public.decoding_test: INSERT: x[integer]:3 y[text]:'3' +table public.decoding_test: INSERT: x[integer]:4 y[text]:'4' +COMMIT}; + +my $stdout_sql = $node_master->safe_psql('postgres', + qq[SELECT data FROM pg_logical_slot_peek_changes('test_slot', NULL, NULL, 'include-xids', '0', 'skip-empty-xacts', '1');], + extra_params => [ '-U', 'normal' ] +); +is($stdout_sql, $expected, 'got expected output from SQL decoding session'); + +my $endpos = $node_master->safe_psql('postgres', + "SELECT lsn FROM pg_logical_slot_peek_changes('test_slot', NULL, NULL) ORDER BY lsn DESC LIMIT 1;", + extra_params => [ '-U', 'normal' ] +); +print "waiting to replay $endpos\n"; + +my $stdout_recv = $node_master->pg_recvlogical_upto( + 'postgres', 'test_slot', $endpos, 180, + 'include-xids' => '0', + 'skip-empty-xacts' => '1'); +chomp($stdout_recv); +is($stdout_recv, $expected, + 'got same expected output from pg_recvlogical decoding session'); + +$node_master->poll_query_until('postgres', + "SELECT EXISTS (SELECT 1 FROM pg_replication_slots WHERE slot_name = 'test_slot' AND active_pid IS NULL)" +) or die "slot never became inactive"; + +$stdout_recv = $node_master->pg_recvlogical_upto( + 'postgres', 'test_slot', $endpos, 180, + 'include-xids' => '0', + 'skip-empty-xacts' => '1'); +chomp($stdout_recv); +is($stdout_recv, '', + 'pg_recvlogical acknowledged changes, nothing pending on slot'); + +$node_master->safe_psql('postgres', 'CREATE DATABASE otherdb'); + +is( $node_master->psql( + 'otherdb', + "SELECT lsn FROM pg_logical_slot_peek_changes('test_slot', NULL, NULL) ORDER BY lsn DESC LIMIT 1;", + extra_params => [ '-U', 'normal' ] + ), + 3, + 'replaying logical slot from another database fails'); + +$node_master->safe_psql('otherdb', + qq[SELECT pg_create_logical_replication_slot('otherdb_slot', 'test_decoding');], + extra_params => [ '-U', 'normal' ] +); + +# make sure you can't drop a slot while active +SKIP: +{ + + # some Windows Perls at least don't like IPC::Run's start/kill_kill regime. + skip "Test fails on Windows perl", 2 if $Config{osname} eq 'MSWin32'; + + my $pg_recvlogical = IPC::Run::start( + [ + 'pg_recvlogical', '-d', $node_master->connstr('otherdb'), + '-S', 'otherdb_slot', '-f', '-', '--start', '-U' , 'normal' + ]); + $node_master->poll_query_until('otherdb', + "SELECT EXISTS (SELECT 1 FROM pg_replication_slots WHERE slot_name = 'otherdb_slot' AND active_pid IS NOT NULL)" + ) or die "slot never became active"; + is($node_master->psql('postgres', 'DROP DATABASE otherdb'), + 3, 'dropping a DB with active logical slots fails'); + $pg_recvlogical->kill_kill; + is($node_master->slot('otherdb_slot')->{'slot_name'}, + undef, 'logical slot still exists'); +} + +$node_master->poll_query_until('otherdb', + "SELECT EXISTS (SELECT 1 FROM pg_replication_slots WHERE slot_name = 'otherdb_slot' AND active_pid IS NULL)", +) or die "slot never became inactive"; + +is($node_master->psql('postgres', 'DROP DATABASE otherdb'), + 0, 'dropping a DB with inactive logical slots succeeds'); +is($node_master->slot('otherdb_slot')->{'slot_name'}, + undef, 'logical slot was actually dropped with DB'); + +# done with the node +$node_master->stop; diff --git a/src/test/regress/expected/mdb_replication.out b/src/test/regress/expected/mdb_replication.out new file mode 100644 index 00000000000..2c8f7bac409 --- /dev/null +++ b/src/test/regress/expected/mdb_replication.out @@ -0,0 +1,35 @@ +CREATE ROLE regress_mdb_repl_no_priv LOGIN NOSUPERUSER; +CREATE ROLE regress_mdb_repl_no_priv2 LOGIN NOSUPERUSER; +CREATE ROLE regress_mdb_repl LOGIN NOSUPERUSER; +CREATE ROLE regress_mdb_repl_su LOGIN SUPERUSER; +CREATE ROLE regress_mdb_repl_pgrad LOGIN NOSUPERUSER; +GRANT pg_read_all_data TO regress_mdb_repl_pgrad; +GRANT mdb_admin to regress_mdb_repl; +GRANT CREATE ON DATABASE REGRESSION TO regress_mdb_repl; +GRANT CREATE ON DATABASE REGRESSION TO regress_mdb_repl_no_priv; +-- ok - member of mdb_admin +SET SESSION AUTHORIZATION regress_mdb_repl; +CREATE SUBSCRIPTION regress_mdbsub CONNECTION 'dbname=doesnotexist password=regress_fakepassword' PUBLICATION foo WITH (slot_name = NONE, connect = false); +WARNING: subscription was created, but is not connected +HINT: To initiate replication, you must manually create the replication slot, enable the subscription, and refresh the subscription. +-- ok - we are allowed to change ownership under mdb_admin +ALTER SUBSCRIPTION regress_mdbsub OWNER TO regress_mdb_repl_no_priv2; +-- should fail - we are not allowed to change ownership to priviledged role +ALTER SUBSCRIPTION regress_mdbsub OWNER TO regress_mdb_repl_su; +ERROR: must be owner of subscription regress_mdbsub +ALTER SUBSCRIPTION regress_mdbsub OWNER TO regress_mdb_repl_pgrad; +ERROR: must be owner of subscription regress_mdbsub +RESET SESSION AUTHORIZATION; +DROP SUBSCRIPTION regress_mdbsub; +-- should fail - not member of pg_subcription_users or mdb_admin +SET SESSION AUTHORIZATION regress_mdb_repl_no_priv; +CREATE SUBSCRIPTION regress_mdbsub_no_priv CONNECTION 'dbname=doesnotexist password=regress_fakepassword' PUBLICATION foo WITH (slot_name = NONE, connect = false); +ERROR: permission denied to create subscription +DETAIL: Only roles with privileges of the "pg_create_subscription" role may create subscriptions. +-- reset to su, cleanup +RESET SESSION AUTHORIZATION; +REVOKE ALL ON DATABASE REGRESSION FROM regress_mdb_repl; +REVOKE ALL ON DATABASE REGRESSION FROM regress_mdb_repl_no_priv; +DROP ROLE regress_mdb_repl; +DROP ROLE regress_mdb_repl_no_priv; +DROP ROLE regress_mdb_repl_no_priv2; diff --git a/src/test/regress/expected/test_setup.out b/src/test/regress/expected/test_setup.out index 882f560eb19..2344180f422 100644 --- a/src/test/regress/expected/test_setup.out +++ b/src/test/regress/expected/test_setup.out @@ -240,3 +240,6 @@ create function fipshash(text) strict immutable parallel safe leakproof return substr(encode(sha256($1::bytea), 'hex'), 1, 32); CREATE ROLE mdb_admin; +CREATE ROLE mdb_superuser; +CREATE ROLE mdb_replication; +GRANT pg_create_subscription TO mdb_admin; diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule index e24513f14ff..79da6b98504 100644 --- a/src/test/regress/parallel_schedule +++ b/src/test/regress/parallel_schedule @@ -15,6 +15,10 @@ test: test_setup test: mdb_admin +test: mdb_superuser + +test: mdb_replication + # ---------- # The first group of parallel tests # ---------- diff --git a/src/test/regress/sql/mdb_replication.sql b/src/test/regress/sql/mdb_replication.sql new file mode 100644 index 00000000000..31bcb460879 --- /dev/null +++ b/src/test/regress/sql/mdb_replication.sql @@ -0,0 +1,41 @@ +CREATE ROLE regress_mdb_repl_no_priv LOGIN NOSUPERUSER; +CREATE ROLE regress_mdb_repl_no_priv2 LOGIN NOSUPERUSER; +CREATE ROLE regress_mdb_repl LOGIN NOSUPERUSER; +CREATE ROLE regress_mdb_repl_su LOGIN SUPERUSER; +CREATE ROLE regress_mdb_repl_pgrad LOGIN NOSUPERUSER; + +GRANT pg_read_all_data TO regress_mdb_repl_pgrad; + +GRANT mdb_admin to regress_mdb_repl; + +GRANT CREATE ON DATABASE REGRESSION TO regress_mdb_repl; + +GRANT CREATE ON DATABASE REGRESSION TO regress_mdb_repl_no_priv; + +-- ok - member of mdb_admin +SET SESSION AUTHORIZATION regress_mdb_repl; +CREATE SUBSCRIPTION regress_mdbsub CONNECTION 'dbname=doesnotexist password=regress_fakepassword' PUBLICATION foo WITH (slot_name = NONE, connect = false); + +-- ok - we are allowed to change ownership under mdb_admin +ALTER SUBSCRIPTION regress_mdbsub OWNER TO regress_mdb_repl_no_priv2; + +-- should fail - we are not allowed to change ownership to priviledged role +ALTER SUBSCRIPTION regress_mdbsub OWNER TO regress_mdb_repl_su; +ALTER SUBSCRIPTION regress_mdbsub OWNER TO regress_mdb_repl_pgrad; + +RESET SESSION AUTHORIZATION; +DROP SUBSCRIPTION regress_mdbsub; + +-- should fail - not member of pg_subcription_users or mdb_admin +SET SESSION AUTHORIZATION regress_mdb_repl_no_priv; +CREATE SUBSCRIPTION regress_mdbsub_no_priv CONNECTION 'dbname=doesnotexist password=regress_fakepassword' PUBLICATION foo WITH (slot_name = NONE, connect = false); +-- reset to su, cleanup + +RESET SESSION AUTHORIZATION; + +REVOKE ALL ON DATABASE REGRESSION FROM regress_mdb_repl; +REVOKE ALL ON DATABASE REGRESSION FROM regress_mdb_repl_no_priv; + +DROP ROLE regress_mdb_repl; +DROP ROLE regress_mdb_repl_no_priv; +DROP ROLE regress_mdb_repl_no_priv2; \ No newline at end of file diff --git a/src/test/regress/sql/test_setup.sql b/src/test/regress/sql/test_setup.sql index 2bdcd9f2161..ef096c6b930 100644 --- a/src/test/regress/sql/test_setup.sql +++ b/src/test/regress/sql/test_setup.sql @@ -296,3 +296,7 @@ create function fipshash(text) return substr(encode(sha256($1::bytea), 'hex'), 1, 32); CREATE ROLE mdb_admin; +CREATE ROLE mdb_superuser; +CREATE ROLE mdb_replication; + +GRANT pg_create_subscription TO mdb_admin; diff --git a/src/test/subscription/t/101_mdb_rep_changes_nonsuperuser.pl b/src/test/subscription/t/101_mdb_rep_changes_nonsuperuser.pl new file mode 100644 index 00000000000..0d3ae411ddf --- /dev/null +++ b/src/test/subscription/t/101_mdb_rep_changes_nonsuperuser.pl @@ -0,0 +1,336 @@ +# Basic logical replication test +use strict; +use warnings; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +# Initialize publisher node +my $node_publisher = PostgreSQL::Test::Cluster->new('publisher'); +$node_publisher->init(allows_streaming => 'logical'); +$node_publisher->start; + +# Create subscriber node +my $node_subscriber = PostgreSQL::Test::Cluster->new('subscriber'); +$node_subscriber->init(allows_streaming => 'logical'); +$node_subscriber->start; + +# Prepare publisher +$node_publisher->safe_psql('postgres', + "CREATE ROLE normal_mdb_repl LOGIN PASSWORD 'regress_lolkekpassword'"); +$node_publisher->safe_psql('postgres', + "CREATE ROLE mdb_replication"); +$node_publisher->safe_psql('postgres', + "GRANT mdb_replication TO normal_mdb_repl"); +$node_publisher->safe_psql('postgres', + "GRANT CREATE ON DATABASE postgres TO normal_mdb_repl"); +$node_publisher->safe_psql('postgres', + "GRANT ALL ON SCHEMA public TO normal_mdb_repl;"); + + +# Delete pg_hba.conf from the given node, add a new entry to it +# and then execute a reload to refresh it. +sub reset_pg_hba +{ + my $node = shift; + my $hba_method = shift; + + unlink($node->data_dir . '/pg_hba.conf'); + $node->append_conf('pg_hba.conf', "local all normal_mdb_repl $hba_method"); + $node->append_conf('pg_hba.conf', "local all all trust"); + $node->reload; + return; +} + +# Prepare subscriber +$node_subscriber->safe_psql('postgres', + "CREATE ROLE normal_mdb_admin LOGIN"); +$node_subscriber->safe_psql('postgres', + "CREATE ROLE mdb_admin"); +$node_subscriber->safe_psql('postgres', + "GRANT pg_create_subscription TO mdb_admin"); +$node_subscriber->safe_psql('postgres', + "GRANT mdb_admin TO normal_mdb_admin"); +$node_subscriber->safe_psql('postgres', + "GRANT CREATE ON DATABASE postgres TO normal_mdb_admin"); +$node_subscriber->safe_psql('postgres', + "GRANT ALL ON SCHEMA public TO normal_mdb_admin;"); + +# Create some preexisting content on publisher +$node_publisher->safe_psql('postgres', + "CREATE TABLE tab_notrep AS SELECT generate_series(1,10) AS a", + extra_params => [ '-U', 'normal_mdb_repl' ]); +$node_publisher->safe_psql('postgres', + "CREATE TABLE tab_ins AS SELECT generate_series(1,1002) AS a", + extra_params => [ '-U', 'normal_mdb_repl' ]); +$node_publisher->safe_psql('postgres', + "CREATE TABLE tab_full AS SELECT generate_series(1,10) AS a", + extra_params => [ '-U', 'normal_mdb_repl' ]); +$node_publisher->safe_psql('postgres', "CREATE TABLE tab_full2 (x text)", + extra_params => [ '-U', 'normal_mdb_repl' ]); +$node_publisher->safe_psql('postgres', + "INSERT INTO tab_full2 VALUES ('a'), ('b'), ('b')", + extra_params => [ '-U', 'normal_mdb_repl' ]); +$node_publisher->safe_psql('postgres', + "CREATE TABLE tab_rep (a int primary key)", + extra_params => [ '-U', 'normal_mdb_repl' ]); +$node_publisher->safe_psql('postgres', + "CREATE TABLE tab_mixed (a int primary key, b text)", + extra_params => [ '-U', 'normal_mdb_repl' ]); +$node_publisher->safe_psql('postgres', + "INSERT INTO tab_mixed (a, b) VALUES (1, 'foo')", + extra_params => [ '-U', 'normal_mdb_repl' ]); +$node_publisher->safe_psql('postgres', + "CREATE TABLE tab_include (a int, b text, CONSTRAINT covering PRIMARY KEY(a) INCLUDE(b))", + extra_params => [ '-U', 'normal_mdb_repl' ] +); + + +# Setup structure on subscriber +$node_subscriber->safe_psql('postgres', "CREATE TABLE tab_notrep (a int)", extra_params => [ '-U', 'normal_mdb_admin' ]); +$node_subscriber->safe_psql('postgres', "CREATE TABLE tab_ins (a int)", extra_params => [ '-U', 'normal_mdb_admin' ]); +$node_subscriber->safe_psql('postgres', "CREATE TABLE tab_full (a int)", extra_params => [ '-U', 'normal_mdb_admin' ]); +$node_subscriber->safe_psql('postgres', "CREATE TABLE tab_full2 (x text)", extra_params => [ '-U', 'normal_mdb_admin' ]); +$node_subscriber->safe_psql('postgres', + "CREATE TABLE tab_rep (a int primary key)", + extra_params => [ '-U', 'normal_mdb_admin' ]); + +# different column count and order than on publisher +$node_subscriber->safe_psql('postgres', + "CREATE TABLE tab_mixed (c text, b text, a int primary key)", + extra_params => [ '-U', 'normal_mdb_admin' ]); + +# replication of the table with included index +$node_subscriber->safe_psql('postgres', + "CREATE TABLE tab_include (a int, b text, CONSTRAINT covering PRIMARY KEY(a) INCLUDE(b))", + extra_params => [ '-U', 'normal_mdb_admin' ] +); + +# Setup logical replication +my $publisher_connstr = $node_publisher->connstr . ' dbname=postgres user=normal_mdb_repl'; +$node_publisher->safe_psql('postgres', "CREATE PUBLICATION tap_pub", extra_params => [ '-U', 'normal_mdb_repl' ]); +$node_publisher->safe_psql('postgres', + "CREATE PUBLICATION tap_pub_ins_only WITH (publish = insert)", + extra_params => [ '-U', 'normal_mdb_repl' ]); +$node_publisher->safe_psql('postgres', + "ALTER PUBLICATION tap_pub ADD TABLE tab_rep, tab_full, tab_full2, tab_mixed, tab_include", + extra_params => [ '-U', 'normal_mdb_repl' ] +); +$node_publisher->safe_psql('postgres', + "ALTER PUBLICATION tap_pub_ins_only ADD TABLE tab_ins", + extra_params => [ '-U', 'normal_mdb_repl' ]); + + +# require password for all connection on publisher +reset_pg_hba($node_publisher, 'scram-sha-256'); + +$ENV{"PGPASSWORD"} = 'regress_lolkekpassword'; + +my $appname = 'tap_sub'; +$node_subscriber->safe_psql('postgres', + "CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr password=regress_lolkekpassword application_name=$appname' PUBLICATION tap_pub, tap_pub_ins_only", + extra_params => [ '-U', 'normal_mdb_admin' ] +); + +$node_publisher->wait_for_catchup($appname); + +# Also wait for initial table sync to finish +my $synced_query = + "SELECT count(1) = 0 FROM pg_subscription_rel WHERE srsubstate NOT IN ('r', 's');"; +$node_subscriber->poll_query_until('postgres', $synced_query) + or die "Timed out while waiting for subscriber to synchronize data"; + +my $result = + $node_subscriber->safe_psql('postgres', "SELECT count(*) FROM tab_notrep"); +is($result, qq(0), 'check non-replicated table is empty on subscriber'); + +$result = + $node_subscriber->safe_psql('postgres', "SELECT count(*) FROM tab_ins"); +is($result, qq(1002), 'check initial data was copied to subscriber'); + +$node_publisher->safe_psql('postgres', + "INSERT INTO tab_ins SELECT generate_series(1,50)"); +$node_publisher->safe_psql('postgres', "DELETE FROM tab_ins WHERE a > 20"); +$node_publisher->safe_psql('postgres', "UPDATE tab_ins SET a = -a"); + +$node_publisher->safe_psql('postgres', + "INSERT INTO tab_rep SELECT generate_series(1,50)"); +$node_publisher->safe_psql('postgres', "DELETE FROM tab_rep WHERE a > 20"); +$node_publisher->safe_psql('postgres', "UPDATE tab_rep SET a = -a"); + +$node_publisher->safe_psql('postgres', + "INSERT INTO tab_mixed VALUES (2, 'bar')"); + +$node_publisher->safe_psql('postgres', + "INSERT INTO tab_include SELECT generate_series(1,50)"); +$node_publisher->safe_psql('postgres', + "DELETE FROM tab_include WHERE a > 20"); +$node_publisher->safe_psql('postgres', "UPDATE tab_include SET a = -a"); + +$node_publisher->wait_for_catchup($appname); + +$result = $node_subscriber->safe_psql('postgres', + "SELECT count(*), min(a), max(a) FROM tab_ins"); +is($result, qq(1052|1|1002), 'check replicated inserts on subscriber'); + +$result = $node_subscriber->safe_psql('postgres', + "SELECT count(*), min(a), max(a) FROM tab_rep"); +is($result, qq(20|-20|-1), 'check replicated changes on subscriber'); + +$result = + $node_subscriber->safe_psql('postgres', "SELECT c, b, a FROM tab_mixed"); +is( $result, qq(|foo|1 +|bar|2), 'check replicated changes with different column order'); + +$result = $node_subscriber->safe_psql('postgres', + "SELECT count(*), min(a), max(a) FROM tab_include"); +is($result, qq(20|-20|-1), + 'check replicated changes with primary key index with included columns'); + +# insert some duplicate rows +$node_publisher->safe_psql('postgres', + "INSERT INTO tab_full SELECT generate_series(1,10)"); + +# add REPLICA IDENTITY FULL so we can update +$node_publisher->safe_psql('postgres', + "ALTER TABLE tab_full REPLICA IDENTITY FULL"); +$node_subscriber->safe_psql('postgres', + "ALTER TABLE tab_full REPLICA IDENTITY FULL"); +$node_publisher->safe_psql('postgres', + "ALTER TABLE tab_full2 REPLICA IDENTITY FULL"); +$node_subscriber->safe_psql('postgres', + "ALTER TABLE tab_full2 REPLICA IDENTITY FULL"); +$node_publisher->safe_psql('postgres', + "ALTER TABLE tab_ins REPLICA IDENTITY FULL"); +$node_subscriber->safe_psql('postgres', + "ALTER TABLE tab_ins REPLICA IDENTITY FULL"); + +# and do the updates +$node_publisher->safe_psql('postgres', "UPDATE tab_full SET a = a * a"); +$node_publisher->safe_psql('postgres', + "UPDATE tab_full2 SET x = 'bb' WHERE x = 'b'"); + +$node_publisher->wait_for_catchup($appname); + +$result = $node_subscriber->safe_psql('postgres', + "SELECT count(*), min(a), max(a) FROM tab_full"); +is($result, qq(20|1|100), + 'update works with REPLICA IDENTITY FULL and duplicate tuples'); + +$result = $node_subscriber->safe_psql('postgres', + "SELECT x FROM tab_full2 ORDER BY 1"); +is( $result, qq(a +bb +bb), + 'update works with REPLICA IDENTITY FULL and text datums'); + +# check that change of connection string and/or publication list causes +# restart of subscription workers. Not all of these are registered as tests +# as we need to poll for a change but the test suite will fail none the less +# when something goes wrong. +my $oldpid = $node_publisher->safe_psql('postgres', + "SELECT pid FROM pg_stat_replication WHERE application_name = '$appname';" +); +$node_subscriber->safe_psql('postgres', + "ALTER SUBSCRIPTION tap_sub CONNECTION 'application_name=$appname $publisher_connstr password=regress_lolkekpassword '", + extra_params => [ '-U', 'normal_mdb_admin' ] +); +$node_publisher->poll_query_until('postgres', + "SELECT pid != $oldpid FROM pg_stat_replication WHERE application_name = '$appname';" +) or die "Timed out while waiting for apply to restart"; + +$oldpid = $node_publisher->safe_psql('postgres', + "SELECT pid FROM pg_stat_replication WHERE application_name = '$appname';" +); +$node_subscriber->safe_psql('postgres', + "ALTER SUBSCRIPTION tap_sub SET PUBLICATION tap_pub_ins_only WITH (copy_data = false)", + extra_params => [ '-U', 'normal_mdb_admin' ] +); +$node_publisher->poll_query_until('postgres', + "SELECT pid != $oldpid FROM pg_stat_replication WHERE application_name = '$appname';" +) or die "Timed out while waiting for apply to restart"; + +$node_publisher->safe_psql('postgres', + "INSERT INTO tab_ins SELECT generate_series(1001,1100)"); +$node_publisher->safe_psql('postgres', "DELETE FROM tab_rep"); + +# Restart the publisher and check the state of the subscriber which +# should be in a streaming state after catching up. +$node_publisher->stop('fast'); +$node_publisher->start; + +$node_publisher->wait_for_catchup($appname); + +$result = $node_subscriber->safe_psql('postgres', + "SELECT count(*), min(a), max(a) FROM tab_ins"); +is($result, qq(1152|1|1100), + 'check replicated inserts after subscription publication change'); + +$result = $node_subscriber->safe_psql('postgres', + "SELECT count(*), min(a), max(a) FROM tab_rep"); +is($result, qq(20|-20|-1), + 'check changes skipped after subscription publication change'); + +# check alter publication (relcache invalidation etc) +$node_publisher->safe_psql('postgres', + "ALTER PUBLICATION tap_pub_ins_only SET (publish = 'insert, delete')"); +$node_publisher->safe_psql('postgres', + "ALTER PUBLICATION tap_pub_ins_only ADD TABLE tab_full"); +$node_publisher->safe_psql('postgres', "DELETE FROM tab_ins WHERE a > 0"); +$node_subscriber->safe_psql('postgres', + "ALTER SUBSCRIPTION tap_sub REFRESH PUBLICATION WITH (copy_data = false)", + extra_params => [ '-U', 'normal_mdb_admin' ] +); +$node_publisher->safe_psql('postgres', "INSERT INTO tab_full VALUES(0)"); + +$node_publisher->wait_for_catchup($appname); + +# note that data are different on provider and subscriber +$result = $node_subscriber->safe_psql('postgres', + "SELECT count(*), min(a), max(a) FROM tab_ins"); +is($result, qq(1052|1|1002), + 'check replicated deletes after alter publication'); + +$result = $node_subscriber->safe_psql('postgres', + "SELECT count(*), min(a), max(a) FROM tab_full"); +is($result, qq(21|0|100), 'check replicated insert after alter publication'); + +# check restart on rename +$oldpid = $node_publisher->safe_psql('postgres', + "SELECT pid FROM pg_stat_replication WHERE application_name = '$appname';" +); +$node_subscriber->safe_psql('postgres', + "ALTER SUBSCRIPTION tap_sub RENAME TO tap_sub_renamed", + extra_params => [ '-U', 'normal_mdb_admin' ]); +$node_publisher->poll_query_until('postgres', + "SELECT pid != $oldpid FROM pg_stat_replication WHERE application_name = '$appname';" +) or die "Timed out while waiting for apply to restart"; + +# check all the cleanup +$node_subscriber->safe_psql('postgres', "DROP SUBSCRIPTION tap_sub_renamed", extra_params => [ '-U', 'normal_mdb_admin' ]); + +$result = $node_subscriber->safe_psql('postgres', + "SELECT count(*) FROM pg_subscription"); +is($result, qq(0), 'check subscription was dropped on subscriber'); + +$result = $node_publisher->safe_psql('postgres', + "SELECT count(*) FROM pg_replication_slots"); +is($result, qq(0), 'check replication slot was dropped on publisher'); + +$result = $node_subscriber->safe_psql('postgres', + "SELECT count(*) FROM pg_subscription_rel"); +is($result, qq(0), + 'check subscription relation status was dropped on subscriber'); + +$result = $node_publisher->safe_psql('postgres', + "SELECT count(*) FROM pg_replication_slots"); +is($result, qq(0), 'check replication slot was dropped on publisher'); + +$result = $node_subscriber->safe_psql('postgres', + "SELECT count(*) FROM pg_replication_origin"); +is($result, qq(0), 'check replication origin was dropped on subscriber'); + +$node_subscriber->stop('fast'); +$node_publisher->stop('fast'); + +done_testing(); diff --git a/src/test/subscription/t/102_mdb_check_permissions.pl b/src/test/subscription/t/102_mdb_check_permissions.pl new file mode 100644 index 00000000000..1efd1cccf6e --- /dev/null +++ b/src/test/subscription/t/102_mdb_check_permissions.pl @@ -0,0 +1,202 @@ + +# Copyright (c) 2021-2023, PostgreSQL Global Development Group + +# Test that logical replication respects permissions +use strict; +use warnings; +use PostgreSQL::Test::Cluster; +use Test::More; + +my ($node_publisher, $node_subscriber, $publisher_connstr, $result, $offset); +$offset = 0; + +sub publish_insert +{ + my ($tbl, $new_i) = @_; + $node_publisher->safe_psql( + 'postgres', qq( + SET SESSION AUTHORIZATION regress_user1; + INSERT INTO $tbl (i) VALUES ($new_i); + )); +} + +sub publish_update +{ + my ($tbl, $old_i, $new_i) = @_; + $node_publisher->safe_psql( + 'postgres', qq( + SET SESSION AUTHORIZATION regress_user1; + UPDATE $tbl SET i = $new_i WHERE i = $old_i; + )); +} + +sub publish_delete +{ + my ($tbl, $old_i) = @_; + $node_publisher->safe_psql( + 'postgres', qq( + SET SESSION AUTHORIZATION regress_user1; + DELETE FROM $tbl WHERE i = $old_i; + )); +} + + +sub publish_truncate +{ + my ($tbl) = @_; + $node_publisher->safe_psql( + 'postgres', qq( + SET SESSION AUTHORIZATION regress_user1; + TRUNCATE $tbl; + )); +} + +sub expect_replication +{ + my ($tbl, $cnt, $min, $max, $testname) = @_; + $node_publisher->wait_for_catchup('app_test_mdb_admin_sub'); + $result = $node_subscriber->safe_psql( + 'postgres', qq( + SELECT COUNT(i), MIN(i), MAX(i) FROM $tbl)); + is($result, "$cnt|$min|$max", $testname); +} + +sub expect_failure +{ + my ($tbl, $cnt, $min, $max, $re, $testname) = @_; + $offset = $node_subscriber->wait_for_log($re, $offset); + $result = $node_subscriber->safe_psql( + 'postgres', qq( + SELECT COUNT(i), MIN(i), MAX(i) FROM $tbl)); + is($result, "$cnt|$min|$max", $testname); +} + +# Create publisher and subscriber nodes with schemas owned and published by +# "regress_alice" but subscribed and replicated by different role +# "regress_admin". For partitioned tables, layout the partitions differently +# on the publisher than on the subscriber. +# +$node_publisher = PostgreSQL::Test::Cluster->new('publisher'); +$node_subscriber = PostgreSQL::Test::Cluster->new('subscriber'); +$node_publisher->init(allows_streaming => 'logical'); +$node_subscriber->init; +$node_publisher->start; +$node_subscriber->start; +$publisher_connstr = $node_publisher->connstr . ' dbname=postgres'; + +for my $node ($node_publisher, $node_subscriber) +{ + $node->safe_psql( + 'postgres', qq( + CREATE ROLE mdb_admin; + GRANT pg_create_subscription TO mdb_admin; + CREATE ROLE mdb_replication; + + CREATE ROLE regress_user1 NOSUPERUSER LOGIN PASSWORD 'regress_lolkekpassword'; + CREATE ROLE regress_user2 NOSUPERUSER LOGIN PASSWORD 'regress_lolkekpassword'; + + GRANT CREATE ON DATABASE postgres TO regress_user1; + GRANT CREATE ON DATABASE postgres TO regress_user2; + SET SESSION AUTHORIZATION regress_user1; + + CREATE SCHEMA sh; + CREATE TABLE sh.tt (i INTEGER); + + ALTER TABLE sh.tt REPLICA IDENTITY FULL; + + GRANT USAGE ON SCHEMA sh TO regress_user2; + GRANT INSERT, SELECT, UPDATE, DELETE, TRUNCATE ON TABLE sh.tt TO regress_user2; + )); +} + +$node_publisher->safe_psql( + 'postgres', qq( +GRANT mdb_replication TO regress_user2; +SET SESSION AUTHORIZATION regress_user1; + +CREATE PUBLICATION tap_pub + FOR TABLE sh.tt; +)); + + +# Delete pg_hba.conf from the given node, add a new entry to it +# and then execute a reload to refresh it. +sub reset_pg_hba +{ + my $node = shift; + my $hba_method = shift; + + unlink($node->data_dir . '/pg_hba.conf'); + $node->append_conf('pg_hba.conf', "local all regress_user2 $hba_method"); + $node->append_conf('pg_hba.conf', "local all all trust"); + $node->reload; + return; +} + +# require password for all connection on publisher +reset_pg_hba($node_publisher, 'scram-sha-256'); + +$ENV{"PGPASSWORD"} = 'regress_lolkekpassword'; + +$node_subscriber->safe_psql( + 'postgres', qq( +GRANT mdb_admin TO regress_user2; +SET SESSION AUTHORIZATION regress_user2; + +CREATE SUBSCRIPTION test_mdb_admin_sub CONNECTION '$publisher_connstr application_name=app_test_mdb_admin_sub user=regress_user2 password=regress_lolkekpassword' PUBLICATION tap_pub WITH(run_as_owner=true); +)); + +# Wait for initial sync to finish +$node_subscriber->wait_for_subscription_sync($node_publisher, 'app_test_mdb_admin_sub'); + +# Verify that "regress_admin" can replicate into the tables +# +publish_insert("sh.tt", 2); +publish_insert("sh.tt", 8); +publish_insert("sh.tt", 16); +expect_replication("sh.tt", 3, 2, 16, + "mdb_admin/mdb_replication role replicates into sh.tt"); + +# check that no replication possible if permission revoked +$node_subscriber->safe_psql( + 'postgres', qq( + REVOKE INSERT ON TABLE sh.tt FROM regress_user2 +)); + + + +publish_insert("sh.tt", 47); + +expect_failure( + "sh.tt", + 3, + 2, + 16, + qr/ERROR: permission denied for table tt/, + "role without INSERT grant for table fails to replicate insert"); + + +# check that no replication possible if permission revoked +$node_subscriber->safe_psql( + 'postgres', qq( + GRANT INSERT ON TABLE sh.tt TO regress_user2 +)); + + +expect_replication("sh.tt", 4, 2, 47, + "mdb_admin/mdb_replication role replicates after grants again"); + +publish_update("sh.tt", 2 => 7); +publish_delete("sh.tt", 16); + +expect_replication("sh.tt", 3, 7, 47, + "mdb_admin/mdb_replication role replicates update/delete into sh.tt"); + +publish_truncate("sh.tt"); + +publish_insert("sh.tt", 47); + +expect_replication("sh.tt", 1, 47, 47, + "mdb_admin/mdb_replication role replicates truncate sh.tt"); + +done_testing(); From a4254250298aa373032dcb6b49bc4e11473322b4 Mon Sep 17 00:00:00 2001 From: reshke kirill Date: Tue, 21 Feb 2023 19:34:14 +0000 Subject: [PATCH 22/54] Role mdb_superuser: feature and regress testsing This commit introduces new mdb internal role mdb_superuser. Role is capaple of: GRANT/REVOKE any set of priviledges to/from any object in database. Has power of pg_database_owner in any database, including: DROP any object in database (except system catalog and stuff) Role is NOT capaple of: Create database, role, extension or alter other roles with such priviledges. Transfer ownership to /pass has_priv of roles: PG_READ_ALL_DATA PG_WRITE_ALL_DATA PG_EXECUTE_SERVER_PROGRAM PG_READ_SERVER_FILES PG_WRITE_SERVER_FILES Fix configure.ac USE_MDBLOCALES option handling Apply autoreconf stuff Set missing ok parameter ito true while acquiring mdb_superuser oid In regress tests, nobody creates mdb_superuser role, so missing ok is fine Fix spelling Applied suggestion Allow mdb_superuser to have power of pg_database_owner Allow mdb_superuser to alter objects and grant ACl to objects, owned by pg_database_owner. Also, when acl check, allow mdb_superuser use pg_database_owner role power to pass check regression test fixes MDB-32132: fix grantor selection for mdb_superuser (#6) --- src/backend/commands/functioncmds.c | 4 +- src/backend/utils/adt/acl.c | 34 +++- src/include/pg_config.h.in | 6 + src/include/utils/acl.h | 1 + src/test/regress/expected/mdb_superuser.out | 129 ++++++++++++++++ src/test/regress/sql/mdb_superuser.sql | 162 ++++++++++++++++++++ 6 files changed, 333 insertions(+), 3 deletions(-) create mode 100644 src/test/regress/expected/mdb_superuser.out create mode 100644 src/test/regress/sql/mdb_superuser.sql diff --git a/src/backend/commands/functioncmds.c b/src/backend/commands/functioncmds.c index 3fec7743124..327ee98ed1f 100644 --- a/src/backend/commands/functioncmds.c +++ b/src/backend/commands/functioncmds.c @@ -1130,7 +1130,7 @@ CreateFunction(ParseState *pstate, CreateFunctionStmt *stmt) */ if (isLeakProof && !superuser()) { - Oid role = get_role_oid("mdb_admin", true); + Oid role = get_role_oid("mdb_admin", true /*if nodoby created mdb_admin role in this database*/); if (!is_member_of_role(GetUserId(), role)) ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), @@ -1419,7 +1419,7 @@ AlterFunction(ParseState *pstate, AlterFunctionStmt *stmt) procForm->proleakproof = boolVal(leakproof_item->arg); if (procForm->proleakproof && !superuser()) { - Oid role = get_role_oid("mdb_admin", true); + Oid role = get_role_oid("mdb_admin", true /*if nodoby created mdb_admin role in this database*/); if (!is_member_of_role(GetUserId(), role)) ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), diff --git a/src/backend/utils/adt/acl.c b/src/backend/utils/adt/acl.c index adc5ea8a2df..c60d22a1ba3 100644 --- a/src/backend/utils/adt/acl.c +++ b/src/backend/utils/adt/acl.c @@ -5176,7 +5176,7 @@ has_privs_of_role_strict(Oid member, Oid role) /* * Check that role is either one of "dangerous" system role -* or has "strict" (not through mdb_admin) +* or has "strict" (not through mdb_admin or mdb_superuser) * privs of this role */ @@ -5204,6 +5204,8 @@ has_privs_of_unwanted_system_role(Oid role) { bool has_privs_of_role(Oid member, Oid role) { + Oid mdb_superuser_roleoid; + /* Fast path for simple case */ if (member == role) return true; @@ -5212,6 +5214,23 @@ has_privs_of_role(Oid member, Oid role) if (superuser_arg(member)) return true; + mdb_superuser_roleoid = get_role_oid("mdb_superuser", true /*if nodoby created mdb_superuser role in this database*/); + + if (is_member_of_role(member, mdb_superuser_roleoid)) { + /* if target role is superuser, disallow */ + if (!superuser_arg(role)) { + /* we want mdb_roles_admin to bypass + * has_priv_of_roles test + * if target role is neither superuser nor + * some dangerous system role + */ + if (!has_privs_of_unwanted_system_role(role)) { + return true; + } + } + } + + /* * Find all the roles that member has the privileges of, including * multi-level recursion, then see if target role is any one of them. @@ -5221,6 +5240,8 @@ has_privs_of_role(Oid member, Oid role) role); } +// -- mdb_superuser patch + // -- non-upstream patch begin /* * Is userId allowed to bypass ownership check @@ -5528,6 +5549,7 @@ select_best_grantor(Oid roleId, AclMode privileges, List *roles_list; int nrights; ListCell *l; + Oid mdb_superuser_roleoid; /* * The object owner is always treated as having all grant options, so if @@ -5542,6 +5564,16 @@ select_best_grantor(Oid roleId, AclMode privileges, return; } + mdb_superuser_roleoid = get_role_oid("mdb_superuser", true /*if nodoby created mdb_superuser role in this database*/); + + if (is_member_of_role(GetUserId(), mdb_superuser_roleoid) + && has_privs_of_role(GetUserId(), ownerId)) { + *grantorId = ownerId; + AclMode mdb_superuser_allowed_privs = needed_goptions; + *grantOptions = mdb_superuser_allowed_privs; + return; + } + /* * Otherwise we have to do a careful search to see if roleId has the * privileges of any suitable role. Note: we can hang onto the result of diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index ee75c7da5ae..6f7707f78cb 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -263,6 +263,9 @@ /* Define to 1 if you have the `m' library (-lm). */ #undef HAVE_LIBM +/* Define to 1 if you have the `mdblocales' library (-lmdblocales). */ +#undef HAVE_LIBMDBLOCALES + /* Define to 1 if you have the `pam' library (-lpam). */ #undef HAVE_LIBPAM @@ -727,6 +730,9 @@ /* Define to 1 to build with LZ4 support. (--with-lz4) */ #undef USE_LZ4 +/* Define to 1 to build with MDB locales. (--with-mdblocales) */ +#undef USE_MDBLOCALES + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/include/utils/acl.h b/src/include/utils/acl.h index fa0cfa4a8db..1887581b6d9 100644 --- a/src/include/utils/acl.h +++ b/src/include/utils/acl.h @@ -212,6 +212,7 @@ extern int aclmembers(const Acl *acl, Oid **roleids); extern bool has_privs_of_role(Oid member, Oid role); extern bool member_can_set_role(Oid member, Oid role); extern void check_can_set_role(Oid member, Oid role); +extern bool has_privs_of_role_strict(Oid member, Oid role); extern bool is_member_of_role(Oid member, Oid role); extern bool is_member_of_role_nosuper(Oid member, Oid role); extern bool is_admin_of_role(Oid member, Oid role); diff --git a/src/test/regress/expected/mdb_superuser.out b/src/test/regress/expected/mdb_superuser.out new file mode 100644 index 00000000000..8d989661434 --- /dev/null +++ b/src/test/regress/expected/mdb_superuser.out @@ -0,0 +1,129 @@ +CREATE ROLE regress_mdb_superuser_user1; +CREATE ROLE regress_mdb_superuser_user2; +CREATE ROLE regress_mdb_superuser_user3; +CREATE ROLE regress_mdb_su_role_o1; +CREATE ROLE regress_mdb_su_role_o2; +GRANT mdb_superuser TO regress_mdb_su_role_o1; +GRANT mdb_admin TO mdb_superuser; +CREATE ROLE regress_superuser WITH SUPERUSER; +GRANT mdb_superuser TO regress_mdb_superuser_user1; +GRANT CREATE ON DATABASE regression TO regress_mdb_superuser_user2; +GRANT CREATE ON DATABASE regression TO regress_mdb_superuser_user3; +GRANT CREATE ON DATABASE regression TO regress_mdb_su_role_o2; +SET ROLE regress_mdb_superuser_user2; +CREATE FUNCTION regress_mdb_superuser_add(integer, integer) RETURNS integer + AS 'SELECT $1 + $2;' + LANGUAGE SQL + IMMUTABLE + RETURNS NULL ON NULL INPUT; +CREATE SCHEMA regress_mdb_superuser_schema; +CREATE TABLE regress_mdb_superuser_schema.regress_mdb_superuser_table(); +CREATE TABLE regress_mdb_superuser_table(); +CREATE VIEW regress_mdb_superuser_view as SELECT 1; +SET ROLE regress_mdb_superuser_user3; +INSERT INTO regress_mdb_superuser_table SELECT * FROM regress_mdb_superuser_table; +ERROR: permission denied for table regress_mdb_superuser_table +SET ROLE regress_mdb_superuser_user1; +-- mdb_superuser can grant to other role +GRANT USAGE, CREATE ON SCHEMA regress_mdb_superuser_schema TO regress_mdb_superuser_user3; +GRANT ALL PRIVILEGES ON TABLE regress_mdb_superuser_table TO regress_mdb_superuser_user3; +REVOKE ALL PRIVILEGES ON TABLE regress_mdb_superuser_table FROM regress_mdb_superuser_user3; +GRANT INSERT, SELECT ON TABLE regress_mdb_superuser_table TO regress_mdb_superuser_user3; +-- grant works +SET ROLE regress_mdb_superuser_user3; +INSERT INTO regress_mdb_superuser_table SELECT * FROM regress_mdb_superuser_table; +SET ROLE mdb_superuser; +-- mdb_superuser drop object of other role +DROP TABLE regress_mdb_superuser_table; +-- mdb admin fails to transfer ownership to superusers and system roles +RESET SESSION AUTHORIZATION; +CREATE TABLE regress_superuser_table(); +SET ROLE pg_read_server_files; +CREATE TABLE regress_pgrsf_table(); +SET ROLE pg_write_server_files; +CREATE TABLE regress_pgwsf_table(); +SET ROLE pg_execute_server_program; +CREATE TABLE regress_pgxsp_table(); +SET ROLE pg_read_all_data; +CREATE TABLE regress_pgrad_table(); +SET ROLE pg_write_all_data; +CREATE TABLE regress_pgrwd_table(); +SET ROLE mdb_superuser; +-- cannot read all data (fail) +SELECT * FROM pg_authid; +ERROR: permission denied for table pg_authid +-- can drop superuser objects, because has power of pg_database_owner +DROP TABLE regress_superuser_table; +DROP TABLE regress_pgrsf_table; +DROP TABLE regress_pgwsf_table; +DROP TABLE regress_pgxsp_table; +DROP TABLE regress_pgrad_table; +DROP TABLE regress_pgrwd_table; +-- does allowed to creare database, role or extension +-- or grant such priviledge +CREATE DATABASE regress_db_fail; +ERROR: permission denied to create database +CREATE ROLE regress_role_fail; +ERROR: permission denied to create role +DETAIL: Only roles with the CREATEROLE attribute may create roles. +ALTER ROLE mdb_superuser WITH CREATEROLE; +ERROR: permission denied to alter role +DETAIL: Only roles with the CREATEROLE attribute and the ADMIN option on role "mdb_superuser" may alter this role. +ALTER ROLE mdb_superuser WITH CREATEDB; +ERROR: permission denied to alter role +DETAIL: Only roles with the CREATEROLE attribute and the ADMIN option on role "mdb_superuser" may alter this role. +ALTER ROLE regress_mdb_superuser_user2 WITH CREATEROLE; +ERROR: permission denied to alter role +DETAIL: Only roles with the CREATEROLE attribute and the ADMIN option on role "regress_mdb_superuser_user2" may alter this role. +ALTER ROLE regress_mdb_superuser_user2 WITH CREATEDB; +ERROR: permission denied to alter role +DETAIL: Only roles with the CREATEROLE attribute and the ADMIN option on role "regress_mdb_superuser_user2" may alter this role. +-- mdb_superuser more powerfull than pg_database_owner +RESET SESSION AUTHORIZATION; +CREATE DATABASE regress_check_owner OWNER regress_mdb_superuser_user2; +\c regress_check_owner; +SET ROLE regress_mdb_superuser_user2; +CREATE SCHEMA regtest; +CREATE TABLE regtest.regtest(); +-- this should fail +SET ROLE regress_mdb_superuser_user3; +GRANT ALL ON TABLE regtest.regtest TO regress_mdb_superuser_user3; +ERROR: permission denied for schema regtest +ALTER TABLE regtest.regtest OWNER TO regress_mdb_superuser_user3; +ERROR: permission denied for schema regtest +SET ROLE regress_mdb_superuser_user1; +GRANT ALL ON TABLE regtest.regtest TO regress_mdb_superuser_user1; +ALTER TABLE regtest.regtest OWNER TO regress_mdb_superuser_user1; +-- Check grantor +SET ROLE regress_mdb_su_role_o2; +CREATE TABLE public.role_o2_t(); +SET ROLE mdb_superuser; +GRANT SELECT ON public.role_o2_t TO regress_mdb_su_role_o1; +SELECT + grantor +from information_schema.role_table_grants +where grantee='regress_mdb_su_role_o1' AND table_name = 'role_o2_t'; + grantor +------------------------ + regress_mdb_su_role_o2 +(1 row) + +\c regression +DROP DATABASE regress_check_owner; +-- end tests +RESET SESSION AUTHORIZATION; +-- +REVOKE CREATE ON DATABASE regression FROM regress_mdb_superuser_user2; +REVOKE CREATE ON DATABASE regression FROM regress_mdb_superuser_user3; +REVOKE CREATE ON DATABASE regression FROM regress_mdb_su_role_o2; +DROP ROLE regress_mdb_su_role_o1; +DROP ROLE regress_mdb_su_role_o2; +DROP VIEW regress_mdb_superuser_view; +DROP FUNCTION regress_mdb_superuser_add; +DROP TABLE regress_mdb_superuser_schema.regress_mdb_superuser_table; +DROP TABLE regress_mdb_superuser_table; +ERROR: table "regress_mdb_superuser_table" does not exist +DROP SCHEMA regress_mdb_superuser_schema; +DROP ROLE regress_mdb_superuser_user1; +DROP ROLE regress_mdb_superuser_user2; +DROP ROLE regress_mdb_superuser_user3; diff --git a/src/test/regress/sql/mdb_superuser.sql b/src/test/regress/sql/mdb_superuser.sql new file mode 100644 index 00000000000..b0e702c630c --- /dev/null +++ b/src/test/regress/sql/mdb_superuser.sql @@ -0,0 +1,162 @@ +CREATE ROLE regress_mdb_superuser_user1; +CREATE ROLE regress_mdb_superuser_user2; +CREATE ROLE regress_mdb_superuser_user3; + +CREATE ROLE regress_mdb_su_role_o1; +CREATE ROLE regress_mdb_su_role_o2; + +GRANT mdb_superuser TO regress_mdb_su_role_o1; + +GRANT mdb_admin TO mdb_superuser; + +CREATE ROLE regress_superuser WITH SUPERUSER; + +GRANT mdb_superuser TO regress_mdb_superuser_user1; + +GRANT CREATE ON DATABASE regression TO regress_mdb_superuser_user2; +GRANT CREATE ON DATABASE regression TO regress_mdb_superuser_user3; +GRANT CREATE ON DATABASE regression TO regress_mdb_su_role_o2; + +SET ROLE regress_mdb_superuser_user2; + +CREATE FUNCTION regress_mdb_superuser_add(integer, integer) RETURNS integer + AS 'SELECT $1 + $2;' + LANGUAGE SQL + IMMUTABLE + RETURNS NULL ON NULL INPUT; + +CREATE SCHEMA regress_mdb_superuser_schema; +CREATE TABLE regress_mdb_superuser_schema.regress_mdb_superuser_table(); +CREATE TABLE regress_mdb_superuser_table(); +CREATE VIEW regress_mdb_superuser_view as SELECT 1; + +SET ROLE regress_mdb_superuser_user3; +INSERT INTO regress_mdb_superuser_table SELECT * FROM regress_mdb_superuser_table; + +SET ROLE regress_mdb_superuser_user1; + +-- mdb_superuser can grant to other role +GRANT USAGE, CREATE ON SCHEMA regress_mdb_superuser_schema TO regress_mdb_superuser_user3; +GRANT ALL PRIVILEGES ON TABLE regress_mdb_superuser_table TO regress_mdb_superuser_user3; +REVOKE ALL PRIVILEGES ON TABLE regress_mdb_superuser_table FROM regress_mdb_superuser_user3; + +GRANT INSERT, SELECT ON TABLE regress_mdb_superuser_table TO regress_mdb_superuser_user3; + +-- grant works +SET ROLE regress_mdb_superuser_user3; +INSERT INTO regress_mdb_superuser_table SELECT * FROM regress_mdb_superuser_table; + +SET ROLE mdb_superuser; + +-- mdb_superuser drop object of other role +DROP TABLE regress_mdb_superuser_table; +-- mdb admin fails to transfer ownership to superusers and system roles + +RESET SESSION AUTHORIZATION; + +CREATE TABLE regress_superuser_table(); + +SET ROLE pg_read_server_files; + +CREATE TABLE regress_pgrsf_table(); + +SET ROLE pg_write_server_files; + +CREATE TABLE regress_pgwsf_table(); + +SET ROLE pg_execute_server_program; + +CREATE TABLE regress_pgxsp_table(); + +SET ROLE pg_read_all_data; + +CREATE TABLE regress_pgrad_table(); + +SET ROLE pg_write_all_data; + +CREATE TABLE regress_pgrwd_table(); + +SET ROLE mdb_superuser; + +-- cannot read all data (fail) +SELECT * FROM pg_authid; + +-- can drop superuser objects, because has power of pg_database_owner +DROP TABLE regress_superuser_table; +DROP TABLE regress_pgrsf_table; +DROP TABLE regress_pgwsf_table; +DROP TABLE regress_pgxsp_table; +DROP TABLE regress_pgrad_table; +DROP TABLE regress_pgrwd_table; + + +-- does allowed to creare database, role or extension +-- or grant such priviledge + +CREATE DATABASE regress_db_fail; +CREATE ROLE regress_role_fail; + +ALTER ROLE mdb_superuser WITH CREATEROLE; +ALTER ROLE mdb_superuser WITH CREATEDB; + +ALTER ROLE regress_mdb_superuser_user2 WITH CREATEROLE; +ALTER ROLE regress_mdb_superuser_user2 WITH CREATEDB; + +-- mdb_superuser more powerfull than pg_database_owner + +RESET SESSION AUTHORIZATION; +CREATE DATABASE regress_check_owner OWNER regress_mdb_superuser_user2; + +\c regress_check_owner; + +SET ROLE regress_mdb_superuser_user2; +CREATE SCHEMA regtest; +CREATE TABLE regtest.regtest(); + +-- this should fail + +SET ROLE regress_mdb_superuser_user3; +GRANT ALL ON TABLE regtest.regtest TO regress_mdb_superuser_user3; +ALTER TABLE regtest.regtest OWNER TO regress_mdb_superuser_user3; + +SET ROLE regress_mdb_superuser_user1; +GRANT ALL ON TABLE regtest.regtest TO regress_mdb_superuser_user1; +ALTER TABLE regtest.regtest OWNER TO regress_mdb_superuser_user1; + +-- Check grantor + +SET ROLE regress_mdb_su_role_o2; + +CREATE TABLE public.role_o2_t(); + +SET ROLE mdb_superuser; + +GRANT SELECT ON public.role_o2_t TO regress_mdb_su_role_o1; + +SELECT + grantor +from information_schema.role_table_grants +where grantee='regress_mdb_su_role_o1' AND table_name = 'role_o2_t'; + +\c regression +DROP DATABASE regress_check_owner; + +-- end tests + +RESET SESSION AUTHORIZATION; +-- +REVOKE CREATE ON DATABASE regression FROM regress_mdb_superuser_user2; +REVOKE CREATE ON DATABASE regression FROM regress_mdb_superuser_user3; +REVOKE CREATE ON DATABASE regression FROM regress_mdb_su_role_o2; + +DROP ROLE regress_mdb_su_role_o1; +DROP ROLE regress_mdb_su_role_o2; + +DROP VIEW regress_mdb_superuser_view; +DROP FUNCTION regress_mdb_superuser_add; +DROP TABLE regress_mdb_superuser_schema.regress_mdb_superuser_table; +DROP TABLE regress_mdb_superuser_table; +DROP SCHEMA regress_mdb_superuser_schema; +DROP ROLE regress_mdb_superuser_user1; +DROP ROLE regress_mdb_superuser_user2; +DROP ROLE regress_mdb_superuser_user3; From 23f3904ffeef94cefe026488ee5b130f8c0a929a Mon Sep 17 00:00:00 2001 From: reshke Date: Tue, 22 Feb 2022 23:26:51 +0300 Subject: [PATCH 23/54] provide [mdb -postgresql] restict grant roles in YC[MDB-16990] --- src/backend/access/common/Makefile | 3 +- src/backend/access/common/yc_checker.c | 21 +++++++++++++ src/backend/commands/user.c | 20 ++++++++++++ src/backend/utils/misc/guc.c | 1 + src/backend/utils/misc/guc_tables.c | 23 ++++++++++++++ src/backend/utils/misc/postgresql.conf.sample | 2 ++ src/include/access/yc_checker.h | 31 +++++++++++++++++++ src/test/modules/test_misc/t/003_check_guc.pl | 2 ++ 8 files changed, 102 insertions(+), 1 deletion(-) create mode 100644 src/backend/access/common/yc_checker.c create mode 100644 src/include/access/yc_checker.h diff --git a/src/backend/access/common/Makefile b/src/backend/access/common/Makefile index e78de312659..399e7e602ce 100644 --- a/src/backend/access/common/Makefile +++ b/src/backend/access/common/Makefile @@ -29,6 +29,7 @@ OBJS = \ toast_compression.o \ toast_internals.o \ tupconvert.o \ - tupdesc.o + tupdesc.o \ + yc_checker.o include $(top_srcdir)/src/backend/common.mk diff --git a/src/backend/access/common/yc_checker.c b/src/backend/access/common/yc_checker.c new file mode 100644 index 00000000000..a7e7822cc2d --- /dev/null +++ b/src/backend/access/common/yc_checker.c @@ -0,0 +1,21 @@ +/*------------------------------------------------------------------------- + * + * yc_checker.c + * yc routines + * + * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * + * IDENTIFICATION + * src/backend/access/common/yc_checker.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" +#include "access/yc_checker.h" + +/* GUC variables */ + +YCGrantCheckerType yc_grant_checker_type = YC_GRANT_CHECKER_OFF; diff --git a/src/backend/commands/user.c b/src/backend/commands/user.c index aed01cf4cae..e2e24813c41 100644 --- a/src/backend/commands/user.c +++ b/src/backend/commands/user.c @@ -16,6 +16,7 @@ #include "access/htup_details.h" #include "access/table.h" #include "access/xact.h" +#include "access/yc_checker.h" #include "catalog/binary_upgrade.h" #include "catalog/catalog.h" #include "catalog/dependency.h" @@ -1689,6 +1690,25 @@ AddRoleMems(Oid currentUserId, const char *rolename, Oid roleid, Assert(list_length(memberSpecs) == list_length(memberIds)); + if (!superuser()) { + switch (yc_grant_checker_type) { + case YC_GRANT_CHECKER_WARN: + ereport(WARNING, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("Roles granted using SQL might be eventually revoked. Please, use Yandex Cloud console, cli or terraform to grant role in postgresql cluster."))); + break; + case YC_GRANT_CHECKER_CRIT: + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("Roles granted using SQL might be eventually revoked. Please, use Yandex Cloud console, cli or terraform to grant role in postgresql cluster."))); + break; + case YC_GRANT_CHECKER_OFF: + default: + break; + + } + } + /* Validate grantor (and resolve implicit grantor if not specified). */ grantorId = check_role_grantor(currentUserId, roleid, grantorId, true); diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index 29a8972214f..1508b37af85 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -3416,6 +3416,7 @@ set_config_with_handle(const char *name, config_handle *handle, void *newextra = NULL; bool prohibitValueChange = false; bool makeDefault; + Oid role; if (elevel == 0) { diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index ea5b4304be7..5d961576a95 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -90,6 +90,10 @@ #include "utils/ps_status.h" #include "utils/xml.h" +/* MDB patch */ +#include "access/yc_checker.h" +/**/ + /* This value is normally passed in from the Makefile */ #ifndef PG_KRB_SRVTAB #define PG_KRB_SRVTAB "" @@ -492,6 +496,15 @@ static const struct config_enum_entry file_extend_method_options[] = { {NULL, 0, false} }; + +static const struct config_enum_entry yc_grant_checker_options[] = { + {"off", YC_GRANT_CHECKER_OFF, false}, + {"warn", YC_GRANT_CHECKER_WARN, false}, + {"crit", YC_GRANT_CHECKER_CRIT, false}, + {NULL, 0, false} +}; + + /* * Options for enum values stored in other modules */ @@ -4825,6 +4838,16 @@ struct config_enum ConfigureNamesEnum[] = NULL, NULL, NULL }, + { + {"ycmdb.yc_grant_checker", PGC_SUSET, CLIENT_CONN_STATEMENT, + gettext_noop("Enables YC MDB runtime checker, which check if user is ok to grant roles to other users."), + NULL + }, + ((int *) &yc_grant_checker_type), + YC_GRANT_CHECKER_OFF, + yc_grant_checker_options, + NULL, NULL, NULL + }, { {"default_transaction_isolation", PGC_USERSET, CLIENT_CONN_STATEMENT, gettext_noop("Sets the transaction isolation level of each new transaction."), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index 92b49c758d2..97c48c17076 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -847,6 +847,8 @@ #synchronous_commit_cancelation = off +#ycmdb.yc_grant_checker = off + #------------------------------------------------------------------------------ # CUSTOMIZED OPTIONS #------------------------------------------------------------------------------ diff --git a/src/include/access/yc_checker.h b/src/include/access/yc_checker.h new file mode 100644 index 00000000000..9c9a6b6ad4c --- /dev/null +++ b/src/include/access/yc_checker.h @@ -0,0 +1,31 @@ +# +/*------------------------------------------------------------------------- + * + * yc_checker.h + * + * Header file for YC MDB specific only GUC variables, + * + * Portions Copyright (c) 1996-2021, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/storage/yc_checker.h + * + *------------------------------------------------------------------------- + */ +#ifndef PG_YC_CHECKER_H +#define PG_YC_CHECKER_H + +/* Possible values for yc_grant_checker_type */ +typedef enum +{ + YC_GRANT_CHECKER_OFF, + YC_GRANT_CHECKER_WARN, + YC_GRANT_CHECKER_CRIT, +} YCGrantCheckerType; + +/* GUC variables */ + +extern PGDLLIMPORT YCGrantCheckerType yc_grant_checker_type; + + +#endif /* PG_YC_CHECKER_H */ diff --git a/src/test/modules/test_misc/t/003_check_guc.pl b/src/test/modules/test_misc/t/003_check_guc.pl index 3ae4bb1cd9d..e754778c1e8 100644 --- a/src/test/modules/test_misc/t/003_check_guc.pl +++ b/src/test/modules/test_misc/t/003_check_guc.pl @@ -72,6 +72,8 @@ } } +push @gucs_in_file, "ycmdb.yc_grant_checker"; + close $contents; # Cross-check that all the GUCs found in the sample file match the ones From 6d5aec28b0231c3d3eb5f3b68e2d3b9262ed1848 Mon Sep 17 00:00:00 2001 From: reshke Date: Wed, 11 May 2022 23:11:42 +0300 Subject: [PATCH 24/54] MDB-16955 : disallow to kill repl mon in cloud --- src/backend/postmaster/bgworker.c | 28 +++++++++++++++++++++++++++ src/backend/storage/ipc/signalfuncs.c | 11 +++++++++++ src/include/postmaster/bgworker.h | 1 + 3 files changed, 40 insertions(+) diff --git a/src/backend/postmaster/bgworker.c b/src/backend/postmaster/bgworker.c index b37ccb85ad6..9e27d70b49d 100644 --- a/src/backend/postmaster/bgworker.c +++ b/src/backend/postmaster/bgworker.c @@ -1320,3 +1320,31 @@ GetBackgroundWorkerTypeByPid(pid_t pid) return result; } + + +bool +GetBackgroundWorkerFindByPidCmp(pid_t pid, const char *target) +{ + int slotno; + bool result = false; + + LWLockAcquire(BackgroundWorkerLock, LW_SHARED); + + for (slotno = 0; slotno < BackgroundWorkerData->total_slots; slotno++) + { + BackgroundWorkerSlot *slot = &BackgroundWorkerData->slot[slotno]; + + if (slot->pid > 0 && slot->pid == pid) + { + if (slot->worker.bgw_type) { + result = strcmp(target, slot->worker.bgw_type) == 0; + } else { + result = false; + } + break; + } + } + + LWLockRelease(BackgroundWorkerLock); + return result; +} diff --git a/src/backend/storage/ipc/signalfuncs.c b/src/backend/storage/ipc/signalfuncs.c index d1a18ca4f18..5fc16d35d0c 100644 --- a/src/backend/storage/ipc/signalfuncs.c +++ b/src/backend/storage/ipc/signalfuncs.c @@ -109,6 +109,17 @@ pg_signal_backend(int pid, int sig) } } + if (!superuser_arg(GetUserId()) && local_beentry != NULL) { + PgBackendStatus *beentry = &local_beentry->backendStatus; + // MDB-16955 : disallow to kill repl mon in cloud + if (beentry != NULL && beentry->st_backendType == B_BG_WORKER) + { + if (GetBackgroundWorkerFindByPidCmp(beentry->st_procpid, "repl_mon")) { + return SIGNAL_BACKEND_NOPERMISSION; + } + } + } + /* * Can the process we just validated above end, followed by the pid being * recycled for a new process, before reaching here? Then we'd be trying diff --git a/src/include/postmaster/bgworker.h b/src/include/postmaster/bgworker.h index 22fc49ec27f..727f143d721 100644 --- a/src/include/postmaster/bgworker.h +++ b/src/include/postmaster/bgworker.h @@ -125,6 +125,7 @@ extern BgwHandleStatus WaitForBackgroundWorkerStartup(BackgroundWorkerHandle *ha extern BgwHandleStatus WaitForBackgroundWorkerShutdown(BackgroundWorkerHandle *); extern const char *GetBackgroundWorkerTypeByPid(pid_t pid); +extern bool GetBackgroundWorkerFindByPidCmp(pid_t pid, const char *target); /* Terminate a bgworker */ extern void TerminateBackgroundWorker(BackgroundWorkerHandle *handle); From 281fe0f2f9bf78af9957757125274561b19300ea Mon Sep 17 00:00:00 2001 From: "Andrey M. Borodin" Date: Mon, 12 Sep 2022 11:47:30 +0500 Subject: [PATCH 25/54] Demonstrate and fix lock of all SQL queries by pg_stat_statements --- .../pg_stat_statements/pg_stat_statements.c | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/contrib/pg_stat_statements/pg_stat_statements.c b/contrib/pg_stat_statements/pg_stat_statements.c index 2030cb12217..82a4141aed7 100644 --- a/contrib/pg_stat_statements/pg_stat_statements.c +++ b/contrib/pg_stat_statements/pg_stat_statements.c @@ -1309,8 +1309,12 @@ pgss_store(const char *query, uint64 queryId, key.queryid = queryId; key.toplevel = (nesting_level == 0); - /* Lookup the hash table entry with shared lock. */ - LWLockAcquire(pgss->lock, LW_SHARED); + /* + * Lookup the hash table entry with shared lock. + * If exclusive lock is taken - just give up. + */ + if (!LWLockConditionalAcquire(pgss->lock, LW_SHARED)) + return; entry = (pgssEntry *) hash_search(pgss_hash, &key, HASH_FIND, NULL); @@ -1335,7 +1339,13 @@ pgss_store(const char *query, uint64 queryId, norm_query = generate_normalized_query(jstate, query, query_location, &query_len); - LWLockAcquire(pgss->lock, LW_SHARED); + /* exclusive lock may be taken while we were doing this */ + /* XXX: Andrey: I'm not sure we should drop here shared lock at all */ + if (!LWLockConditionalAcquire(pgss->lock, LW_SHARED)) + { + pfree(norm_query); + return; + } } /* Append new query text to file with only shared lock held */ @@ -1351,7 +1361,9 @@ pgss_store(const char *query, uint64 queryId, /* Need exclusive lock to make a new hashtable entry - promote */ LWLockRelease(pgss->lock); - LWLockAcquire(pgss->lock, LW_EXCLUSIVE); + /* This renders impossible to enter another concurrent query */ + if (!LWLockConditionalAcquire(pgss->lock, LW_EXCLUSIVE)) + return; /* * A garbage collection may have occurred while we weren't holding the From 901e77105a219381f88646db288f91ae21a43d6b Mon Sep 17 00:00:00 2001 From: reshke kirill Date: Wed, 18 Jan 2023 18:27:40 +0000 Subject: [PATCH 26/54] MDB-21297: forbit usage of COPY TO PROGRAMM and COPY FROM PROGRAMM to non-superuser It is well known that some of PostgreSQL-related security issues (CVE) was related to COPY FROM/TO PROGRAM exploits for priviledge escalation or other unwanted behaviour or consequences. Thus, proper usage of this feature needed. For now, simply forbit this. Add mdb copy test --- src/backend/commands/copy.c | 29 ++++++++++++------------ src/test/regress/expected/mdb_copy.out | 29 ++++++++++++++++++++++++ src/test/regress/parallel_schedule | 2 ++ src/test/regress/sql/mdb_copy.sql | 31 ++++++++++++++++++++++++++ 4 files changed, 77 insertions(+), 14 deletions(-) create mode 100644 src/test/regress/expected/mdb_copy.out create mode 100644 src/test/regress/sql/mdb_copy.sql diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c index f9ec035aa99..f0e9d6072e0 100644 --- a/src/backend/commands/copy.c +++ b/src/backend/commands/copy.c @@ -78,37 +78,38 @@ DoCopy(ParseState *pstate, const CopyStmt *stmt, { if (stmt->is_program) { - if (!has_privs_of_role(GetUserId(), ROLE_PG_EXECUTE_SERVER_PROGRAM)) - ereport(ERROR, - (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), - errmsg("permission denied to COPY to or from an external program"), - errdetail("Only roles with privileges of the \"%s\" role may COPY to or from an external program.", - "pg_execute_server_program"), - errhint("Anyone can COPY to stdout or from stdin. " - "psql's \\copy command also works for anyone."))); + // -- non-upstream patch begin + /* + * MDB-21297: forbit usage of COPY TO PROGRAM and COPY FROM PROGRAM to non-su + */ + + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("forbidden to COPY to or from an external program or file in Yandex Cloud"), + errhint("Anyone can COPY to stdout or from stdin. " + "psql's \\copy command also works for anyone."))); + + // --- non-upstream patch end } else { if (is_from && !has_privs_of_role(GetUserId(), ROLE_PG_READ_SERVER_FILES)) ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), - errmsg("permission denied to COPY from a file"), - errdetail("Only roles with privileges of the \"%s\" role may COPY from a file.", - "pg_read_server_files"), + errmsg("must be superuser or have privileges of the pg_read_server_files role to COPY from a file"), errhint("Anyone can COPY to stdout or from stdin. " "psql's \\copy command also works for anyone."))); if (!is_from && !has_privs_of_role(GetUserId(), ROLE_PG_WRITE_SERVER_FILES)) ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), - errmsg("permission denied to COPY to a file"), - errdetail("Only roles with privileges of the \"%s\" role may COPY to a file.", - "pg_write_server_files"), + errmsg("must be superuser or have privileges of the pg_write_server_files role to COPY to a file"), errhint("Anyone can COPY to stdout or from stdin. " "psql's \\copy command also works for anyone."))); } } + if (stmt->relation) { LOCKMODE lockmode = is_from ? RowExclusiveLock : AccessShareLock; diff --git a/src/test/regress/expected/mdb_copy.out b/src/test/regress/expected/mdb_copy.out new file mode 100644 index 00000000000..df6d70d1e7d --- /dev/null +++ b/src/test/regress/expected/mdb_copy.out @@ -0,0 +1,29 @@ +CREATE ROLE regress_mdb_copy_r1 LOGIN NOSUPERUSER; +CREATE ROLE regress_mdb_copy_r1_mdb_adm LOGIN NOSUPERUSER; +GRANT mdb_admin TO regress_mdb_copy_r1_mdb_adm; +CREATE ROLE regress_mdb_copy_r1_su LOGIN SUPERUSER; +-- should fail +SET ROLE regress_mdb_copy_r1; +CREATE TABLE tt(i int); +COPY tt FROM PROGRAM '/bin/bash'; +ERROR: forbidden to COPY to or from an external program or file in Yandex Cloud +HINT: Anyone can COPY to stdout or from stdin. psql's \copy command also works for anyone. +DROP TABLE tt; +-- should fail +SET ROLE regress_mdb_copy_r1_mdb_adm; +CREATE TABLE tt(i int); +COPY tt FROM PROGRAM '/bin/bash'; +ERROR: forbidden to COPY to or from an external program or file in Yandex Cloud +HINT: Anyone can COPY to stdout or from stdin. psql's \copy command also works for anyone. +DROP TABLE tt; +-- fail, no one can do it +SET ROLE regress_mdb_copy_r1_su; +CREATE TABLE tt(i int); +COPY tt FROM PROGRAM '/bin/bash'; +ERROR: forbidden to COPY to or from an external program or file in Yandex Cloud +HINT: Anyone can COPY to stdout or from stdin. psql's \copy command also works for anyone. +DROP TABLE tt; +RESET SESSION AUTHORIZATION; +DROP ROLE regress_mdb_copy_r1; +DROP ROLE regress_mdb_copy_r1_mdb_adm; +DROP ROLE regress_mdb_copy_r1_su; diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule index 79da6b98504..6a6574036c8 100644 --- a/src/test/regress/parallel_schedule +++ b/src/test/regress/parallel_schedule @@ -19,6 +19,8 @@ test: mdb_superuser test: mdb_replication +test: mdb_copy + # ---------- # The first group of parallel tests # ---------- diff --git a/src/test/regress/sql/mdb_copy.sql b/src/test/regress/sql/mdb_copy.sql new file mode 100644 index 00000000000..c734b2814c5 --- /dev/null +++ b/src/test/regress/sql/mdb_copy.sql @@ -0,0 +1,31 @@ +CREATE ROLE regress_mdb_copy_r1 LOGIN NOSUPERUSER; +CREATE ROLE regress_mdb_copy_r1_mdb_adm LOGIN NOSUPERUSER; +GRANT mdb_admin TO regress_mdb_copy_r1_mdb_adm; +CREATE ROLE regress_mdb_copy_r1_su LOGIN SUPERUSER; + +-- should fail +SET ROLE regress_mdb_copy_r1; + +CREATE TABLE tt(i int); +COPY tt FROM PROGRAM '/bin/bash'; +DROP TABLE tt; + +-- should fail +SET ROLE regress_mdb_copy_r1_mdb_adm; + +CREATE TABLE tt(i int); +COPY tt FROM PROGRAM '/bin/bash'; +DROP TABLE tt; + +-- fail, no one can do it +SET ROLE regress_mdb_copy_r1_su; + +CREATE TABLE tt(i int); +COPY tt FROM PROGRAM '/bin/bash'; +DROP TABLE tt; + +RESET SESSION AUTHORIZATION; + +DROP ROLE regress_mdb_copy_r1; +DROP ROLE regress_mdb_copy_r1_mdb_adm; +DROP ROLE regress_mdb_copy_r1_su; From 5210a6854bef21280d9f510037ab6580cd024213 Mon Sep 17 00:00:00 2001 From: usernamedt Date: Mon, 13 Feb 2023 15:00:31 +0800 Subject: [PATCH 27/54] Implement mdb-locales patch Do not use mdb_locales and mdb_newlocale when configured without them. Added ifdef codepath build without mdb-locales feature Add mdb locales patch, restore COPY from/to files, enable regress. Squashed commit of the following: commit 9f8ea4a5f42e0fd6077061bafbb428e88499896f Author: reshke kirill Date: Wed Feb 22 09:27:53 2023 +0000 Add mdb locales patch, restore COPY from/to files, enable regress. This commit does several things. * Enables back COPY from/to FILE functionality, becuase it is used by pg_regress * Enables pg_regress tests in deb build itself. * Add mdb locales function and checks that package build with mdb-locales support Add configure ac target to define USE_MDBLOCALES properly Refactor optional setlocale, fix minor issues --- configure | 126 +++++++++++++++++++++-- configure.ac | 17 +++ src/backend/utils/adt/Makefile | 3 +- src/backend/utils/adt/mdb.c | 36 +++++++ src/backend/utils/adt/pg_locale.c | 63 +++++++----- src/backend/utils/mb/mbutils.c | 3 +- src/bin/initdb/initdb.c | 11 +- src/bin/pg_upgrade/check.c | 2 + src/common/exec.c | 4 +- src/include/catalog/pg_proc.dat | 6 +- src/include/common/mdb_locale.h | 24 +++++ src/include/pg_config.h.in | 3 + src/interfaces/ecpg/ecpglib/connect.c | 3 +- src/interfaces/ecpg/ecpglib/descriptor.c | 8 +- src/interfaces/ecpg/ecpglib/execute.c | 7 +- src/interfaces/libpq/Makefile | 2 +- src/pl/plperl/plperl.c | 19 ++-- src/port/chklocale.c | 10 +- src/test/locale/test-ctype.c | 4 +- src/test/regress/expected/misc.out | 7 ++ src/test/regress/sql/misc.sql | 5 + 21 files changed, 297 insertions(+), 66 deletions(-) create mode 100644 src/backend/utils/adt/mdb.c create mode 100644 src/include/common/mdb_locale.h diff --git a/configure b/configure index bf6c773b5dd..d6f2bfedaf2 100755 --- a/configure +++ b/configure @@ -687,6 +687,7 @@ BISON MKDIR_P LN_S TAR +USE_MDBLOCALES install_bin INSTALL_DATA INSTALL_SCRIPT @@ -805,6 +806,7 @@ infodir docdir oldincludedir includedir +runstatedir localstatedir sharedstatedir sysconfdir @@ -874,6 +876,8 @@ with_system_tzdata with_zlib with_lz4 with_zstd +with_gnu_ld +with_mdblocales with_ssl with_openssl enable_largefile @@ -948,6 +952,7 @@ datadir='${datarootdir}' sysconfdir='${prefix}/etc' sharedstatedir='${prefix}/com' localstatedir='${prefix}/var' +runstatedir='${localstatedir}/run' includedir='${prefix}/include' oldincludedir='/usr/include' docdir='${datarootdir}/doc/${PACKAGE_TARNAME}' @@ -1200,6 +1205,15 @@ do | -silent | --silent | --silen | --sile | --sil) silent=yes ;; + -runstatedir | --runstatedir | --runstatedi | --runstated \ + | --runstate | --runstat | --runsta | --runst | --runs \ + | --run | --ru | --r) + ac_prev=runstatedir ;; + -runstatedir=* | --runstatedir=* | --runstatedi=* | --runstated=* \ + | --runstate=* | --runstat=* | --runsta=* | --runst=* | --runs=* \ + | --run=* | --ru=* | --r=*) + runstatedir=$ac_optarg ;; + -sbindir | --sbindir | --sbindi | --sbind | --sbin | --sbi | --sb) ac_prev=sbindir ;; -sbindir=* | --sbindir=* | --sbindi=* | --sbind=* | --sbin=* \ @@ -1337,7 +1351,7 @@ fi for ac_var in exec_prefix prefix bindir sbindir libexecdir datarootdir \ datadir sysconfdir sharedstatedir localstatedir includedir \ oldincludedir docdir infodir htmldir dvidir pdfdir psdir \ - libdir localedir mandir + libdir localedir mandir runstatedir do eval ac_val=\$$ac_var # Remove trailing slashes. @@ -1490,6 +1504,7 @@ Fine tuning of the installation directories: --sysconfdir=DIR read-only single-machine data [PREFIX/etc] --sharedstatedir=DIR modifiable architecture-independent data [PREFIX/com] --localstatedir=DIR modifiable single-machine data [PREFIX/var] + --runstatedir=DIR modifiable per-process data [LOCALSTATEDIR/run] --libdir=DIR object code libraries [EPREFIX/lib] --includedir=DIR C header files [PREFIX/include] --oldincludedir=DIR C header files for non-gcc [/usr/include] @@ -1586,6 +1601,8 @@ Optional Packages: --without-zlib do not use Zlib --with-lz4 build with LZ4 support --with-zstd build with ZSTD support + --with-gnu-ld assume the C compiler uses GNU ld [default=no] + --without-mdblocales build without MDB locales --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -2794,7 +2811,6 @@ ac_compiler_gnu=$ac_cv_c_compiler_gnu - ac_aux_dir= for ac_dir in config "$srcdir"/config; do if test -f "$ac_dir/install-sh"; then @@ -2825,6 +2841,7 @@ ac_configure="$SHELL $ac_aux_dir/configure" # Please don't use this var. +ac_configure_args=$(echo "$ac_configure_args" | sed -e "s/ -f\(debug\|file\)-prefix-map=[^' ]*//g") cat >>confdefs.h <<_ACEOF #define CONFIGURE_ARGS "$ac_configure_args" @@ -9820,6 +9837,40 @@ case $INSTALL in esac +# +# MDB locales +# + + + + +# Check whether --with-mdblocales was given. +if test "${with_mdblocales+set}" = set; then : + withval=$with_mdblocales; + case $withval in + yes) + +$as_echo "#define USE_MDBLOCALES 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-mdblocales option" "$LINENO" 5 + ;; + esac + +else + with_mdblocales=yes + +$as_echo "#define USE_MDBLOCALES 1" >>confdefs.h + +fi + + + + if test -z "$TAR"; then for ac_prog in tar do @@ -12290,6 +12341,56 @@ fi fi +if test "$with_mdblocales" = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for mdb_setlocale in -lmdblocales" >&5 +$as_echo_n "checking for mdb_setlocale in -lmdblocales... " >&6; } +if ${ac_cv_lib_mdblocales_mdb_setlocale+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_check_lib_save_LIBS=$LIBS +LIBS="-lmdblocales $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char mdb_setlocale (); +int +main () +{ +return mdb_setlocale (); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + ac_cv_lib_mdblocales_mdb_setlocale=yes +else + ac_cv_lib_mdblocales_mdb_setlocale=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_mdblocales_mdb_setlocale" >&5 +$as_echo "$ac_cv_lib_mdblocales_mdb_setlocale" >&6; } +if test "x$ac_cv_lib_mdblocales_mdb_setlocale" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LIBMDBLOCALES 1 +_ACEOF + + LIBS="-lmdblocales $LIBS" + +else + as_fn_error $? "mdblocales library not found" "$LINENO" 5 +fi + +fi + if test "$enable_spinlocks" = yes; then $as_echo "#define HAVE_SPINLOCKS 1" >>confdefs.h @@ -13659,6 +13760,17 @@ else fi +fi + +if test "$with_mdblocales" = yes; then + ac_fn_c_check_header_mongrel "$LINENO" "mdblocales.h" "ac_cv_header_mdblocales_h" "$ac_includes_default" +if test "x$ac_cv_header_mdblocales_h" = xyes; then : + +else + as_fn_error $? "mdblocales header not found." "$LINENO" 5 +fi + + fi if test "$with_gssapi" = yes ; then @@ -15000,7 +15112,7 @@ else We can't simply define LARGE_OFF_T to be 9223372036854775807, since some C++ compilers masquerading as C compilers incorrectly reject 9223372036854775807. */ -#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62)) +#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31)) int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721 && LARGE_OFF_T % 2147483647 == 1) ? 1 : -1]; @@ -15046,7 +15158,7 @@ else We can't simply define LARGE_OFF_T to be 9223372036854775807, since some C++ compilers masquerading as C compilers incorrectly reject 9223372036854775807. */ -#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62)) +#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31)) int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721 && LARGE_OFF_T % 2147483647 == 1) ? 1 : -1]; @@ -15070,7 +15182,7 @@ rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext We can't simply define LARGE_OFF_T to be 9223372036854775807, since some C++ compilers masquerading as C compilers incorrectly reject 9223372036854775807. */ -#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62)) +#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31)) int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721 && LARGE_OFF_T % 2147483647 == 1) ? 1 : -1]; @@ -15115,7 +15227,7 @@ else We can't simply define LARGE_OFF_T to be 9223372036854775807, since some C++ compilers masquerading as C compilers incorrectly reject 9223372036854775807. */ -#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62)) +#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31)) int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721 && LARGE_OFF_T % 2147483647 == 1) ? 1 : -1]; @@ -15139,7 +15251,7 @@ rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext We can't simply define LARGE_OFF_T to be 9223372036854775807, since some C++ compilers masquerading as C compilers incorrectly reject 9223372036854775807. */ -#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62)) +#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31)) int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721 && LARGE_OFF_T % 2147483647 == 1) ? 1 : -1]; diff --git a/configure.ac b/configure.ac index 86ed594430a..2ba1fddc24e 100644 --- a/configure.ac +++ b/configure.ac @@ -1154,6 +1154,14 @@ case $INSTALL in esac AC_SUBST(install_bin) +# +# MDB locales +# + +PGAC_ARG_BOOL(with, mdblocales, yes, [build without MDB locales], + [AC_DEFINE([USE_MDBLOCALES], 1, [Define to 1 to build with MDB locales. (--with-mdblocales)])]) +AC_SUBST(USE_MDBLOCALES) + PGAC_PATH_PROGS(TAR, tar) AC_PROG_LN_S AC_PROG_MKDIR_P @@ -1304,6 +1312,11 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_mdblocales" = yes; then + AC_CHECK_LIB(mdblocales, mdb_setlocale, [], + [AC_MSG_ERROR([mdblocales library not found])]) +fi + if test "$enable_spinlocks" = yes; then AC_DEFINE(HAVE_SPINLOCKS, 1, [Define to 1 if you have spinlocks.]) else @@ -1535,6 +1548,10 @@ if test "$with_zstd" = yes; then AC_CHECK_HEADER(zstd.h, [], [AC_MSG_ERROR([zstd.h header file is required for ZSTD])]) fi +if test "$with_mdblocales" = yes; then + AC_CHECK_HEADER(mdblocales.h, [], [AC_MSG_ERROR([mdblocales header not found.])]) +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/src/backend/utils/adt/Makefile b/src/backend/utils/adt/Makefile index edb09d4e356..c58bf38d042 100644 --- a/src/backend/utils/adt/Makefile +++ b/src/backend/utils/adt/Makefile @@ -120,7 +120,8 @@ OBJS = \ windowfuncs.o \ xid.o \ xid8funcs.o \ - xml.o + xml.o \ + mdb.o # See notes in src/backend/parser/Makefile about the following two rules jsonpath_gram.h: jsonpath_gram.c diff --git a/src/backend/utils/adt/mdb.c b/src/backend/utils/adt/mdb.c new file mode 100644 index 00000000000..cc61073fa58 --- /dev/null +++ b/src/backend/utils/adt/mdb.c @@ -0,0 +1,36 @@ +/*------------------------------------------------------------------------- + * + * mdb.c + * mdb routines + * + * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * + * IDENTIFICATION + * src/backend/utils/adt/mdb.c + * + *------------------------------------------------------------------------- + */ + + +#include "postgres.h" +#include "fmgr.h" + +/* + * mdb_admin_enabled + * Check that mdb locale patch is enabled + */ +Datum +mdb_locale_enabled(PG_FUNCTION_ARGS) +{ + bool res; + +#if USE_MDBLOCALES + res = true; +#else + res = false; +#endif + + PG_RETURN_BOOL(res); +} diff --git a/src/backend/utils/adt/pg_locale.c b/src/backend/utils/adt/pg_locale.c index aa0114ce120..07e78645f55 100644 --- a/src/backend/utils/adt/pg_locale.c +++ b/src/backend/utils/adt/pg_locale.c @@ -68,6 +68,7 @@ #include "utils/pg_locale.h" #include "utils/relcache.h" #include "utils/syscache.h" +#include "common/mdb_locale.h" #ifdef USE_ICU #include @@ -217,7 +218,7 @@ pg_perm_setlocale(int category, const char *locale) const char *envvar; #ifndef WIN32 - result = setlocale(category, locale); + result = SETLOCALE(category, locale); #else /* @@ -235,7 +236,7 @@ pg_perm_setlocale(int category, const char *locale) } else #endif - result = setlocale(category, locale); + result = SETLOCALE(category, locale); #endif /* WIN32 */ if (result == NULL) @@ -332,7 +333,7 @@ check_locale(int category, const char *locale, char **canonname) if (canonname) *canonname = NULL; /* in case of failure */ - save = setlocale(category, NULL); + save = SETLOCALE(category, NULL); if (!save) return false; /* won't happen, we hope */ @@ -340,14 +341,14 @@ check_locale(int category, const char *locale, char **canonname) save = pstrdup(save); /* set the locale with setlocale, to see if it accepts it. */ - res = setlocale(category, locale); + res = SETLOCALE(category, locale); /* save canonical name if requested. */ if (res && canonname) *canonname = pstrdup(res); /* restore old value. */ - if (!setlocale(category, save)) + if (!SETLOCALE(category, save)) elog(WARNING, "failed to restore old locale \"%s\"", save); pfree(save); @@ -583,12 +584,12 @@ PGLC_localeconv(void) memset(&worklconv, 0, sizeof(worklconv)); /* Save prevailing values of monetary and numeric locales */ - save_lc_monetary = setlocale(LC_MONETARY, NULL); + save_lc_monetary = SETLOCALE(LC_MONETARY, NULL); if (!save_lc_monetary) elog(ERROR, "setlocale(NULL) failed"); save_lc_monetary = pstrdup(save_lc_monetary); - save_lc_numeric = setlocale(LC_NUMERIC, NULL); + save_lc_numeric = SETLOCALE(LC_NUMERIC, NULL); if (!save_lc_numeric) elog(ERROR, "setlocale(NULL) failed"); save_lc_numeric = pstrdup(save_lc_numeric); @@ -610,7 +611,7 @@ PGLC_localeconv(void) */ /* Save prevailing value of ctype locale */ - save_lc_ctype = setlocale(LC_CTYPE, NULL); + save_lc_ctype = SETLOCALE(LC_CTYPE, NULL); if (!save_lc_ctype) elog(ERROR, "setlocale(NULL) failed"); save_lc_ctype = pstrdup(save_lc_ctype); @@ -618,11 +619,11 @@ PGLC_localeconv(void) /* Here begins the critical section where we must not throw error */ /* use numeric to set the ctype */ - setlocale(LC_CTYPE, locale_numeric); + SETLOCALE(LC_CTYPE, locale_numeric); #endif /* Get formatting information for numeric */ - setlocale(LC_NUMERIC, locale_numeric); + SETLOCALE(LC_NUMERIC, locale_numeric); extlconv = localeconv(); /* Must copy data now in case setlocale() overwrites it */ @@ -632,11 +633,11 @@ PGLC_localeconv(void) #ifdef WIN32 /* use monetary to set the ctype */ - setlocale(LC_CTYPE, locale_monetary); + SETLOCALE(LC_CTYPE, locale_monetary); #endif /* Get formatting information for monetary */ - setlocale(LC_MONETARY, locale_monetary); + SETLOCALE(LC_MONETARY, locale_monetary); extlconv = localeconv(); /* Must copy data now in case setlocale() overwrites it */ @@ -666,12 +667,12 @@ PGLC_localeconv(void) * should fail. */ #ifdef WIN32 - if (!setlocale(LC_CTYPE, save_lc_ctype)) + if (!SETLOCALE(LC_CTYPE, save_lc_ctype)) elog(FATAL, "failed to restore LC_CTYPE to \"%s\"", save_lc_ctype); #endif - if (!setlocale(LC_MONETARY, save_lc_monetary)) + if (!SETLOCALE(LC_MONETARY, save_lc_monetary)) elog(FATAL, "failed to restore LC_MONETARY to \"%s\"", save_lc_monetary); - if (!setlocale(LC_NUMERIC, save_lc_numeric)) + if (!SETLOCALE(LC_NUMERIC, save_lc_numeric)) elog(FATAL, "failed to restore LC_NUMERIC to \"%s\"", save_lc_numeric); /* @@ -855,7 +856,7 @@ cache_locale_time(void) */ /* Save prevailing value of time locale */ - save_lc_time = setlocale(LC_TIME, NULL); + save_lc_time = SETLOCALE(LC_TIME, NULL); if (!save_lc_time) elog(ERROR, "setlocale(NULL) failed"); save_lc_time = pstrdup(save_lc_time); @@ -870,16 +871,16 @@ cache_locale_time(void) */ /* Save prevailing value of ctype locale */ - save_lc_ctype = setlocale(LC_CTYPE, NULL); + save_lc_ctype = SETLOCALE(LC_CTYPE, NULL); if (!save_lc_ctype) elog(ERROR, "setlocale(NULL) failed"); save_lc_ctype = pstrdup(save_lc_ctype); /* use lc_time to set the ctype */ - setlocale(LC_CTYPE, locale_time); + SETLOCALE(LC_CTYPE, locale_time); #endif - setlocale(LC_TIME, locale_time); + SETLOCALE(LC_TIME, locale_time); /* We use times close to current time as data for strftime(). */ timenow = time(NULL); @@ -928,10 +929,10 @@ cache_locale_time(void) * failure to do so is fatal. */ #ifdef WIN32 - if (!setlocale(LC_CTYPE, save_lc_ctype)) + if (!SETLOCALE(LC_CTYPE, save_lc_ctype)) elog(FATAL, "failed to restore LC_CTYPE to \"%s\"", save_lc_ctype); #endif - if (!setlocale(LC_TIME, save_lc_time)) + if (!SETLOCALE(LC_TIME, save_lc_time)) elog(FATAL, "failed to restore LC_TIME to \"%s\"", save_lc_time); /* @@ -1373,7 +1374,7 @@ lc_collate_is_c(Oid collation) } else if (default_locale.provider == COLLPROVIDER_LIBC) { - localeptr = setlocale(LC_COLLATE, NULL); + localeptr = SETLOCALE(LC_COLLATE, NULL); if (!localeptr) elog(ERROR, "invalid LC_COLLATE setting"); } @@ -1439,7 +1440,7 @@ lc_ctype_is_c(Oid collation) } else if (default_locale.provider == COLLPROVIDER_LIBC) { - localeptr = setlocale(LC_CTYPE, NULL); + localeptr = SETLOCALE(LC_CTYPE, NULL); if (!localeptr) elog(ERROR, "invalid LC_CTYPE setting"); } @@ -1642,8 +1643,10 @@ pg_newlocale_from_collation(Oid collid) /* Normal case where they're the same */ errno = 0; #ifndef WIN32 - loc = newlocale(LC_COLLATE_MASK | LC_CTYPE_MASK, collcollate, + + loc = NEWLOCALE(LC_COLLATE_MASK | LC_CTYPE_MASK, collcollate, NULL); + #else loc = _create_locale(LC_ALL, collcollate); #endif @@ -1657,11 +1660,11 @@ pg_newlocale_from_collation(Oid collid) locale_t loc1; errno = 0; - loc1 = newlocale(LC_COLLATE_MASK, collcollate, NULL); + loc1 = NEWLOCALE(LC_COLLATE_MASK, collcollate, NULL); if (!loc1) report_newlocale_failure(collcollate); errno = 0; - loc = newlocale(LC_CTYPE_MASK, collctype, loc1); + loc = NEWLOCALE(LC_CTYPE_MASK, collctype, loc1); if (!loc) report_newlocale_failure(collctype); #else @@ -1803,12 +1806,16 @@ get_collation_actual_version(char collprovider, const char *collcollate) { #if defined(__GLIBC__) /* Use the glibc version because we don't have anything better. */ - collversion = pstrdup(gnu_get_libc_version()); +#ifdef USE_MDBLOCALES + collversion = pstrdup(mdb_localesversion()); +#else + collversion = pstrdup(gnu_get_libc_version()); +#endif #elif defined(LC_VERSION_MASK) locale_t loc; /* Look up FreeBSD collation version. */ - loc = newlocale(LC_COLLATE_MASK, collcollate, NULL); + loc = NEWLOCALE(LC_COLLATE_MASK, collcollate, NULL); if (loc) { collversion = diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c index 271a349a5bd..65cd90e6fb2 100644 --- a/src/backend/utils/mb/mbutils.c +++ b/src/backend/utils/mb/mbutils.c @@ -42,6 +42,7 @@ #include "utils/memutils.h" #include "utils/relcache.h" #include "varatt.h" +#include "common/mdb_locale.h" /* * We maintain a simple linked list caching the fmgr lookup info for the @@ -1356,7 +1357,7 @@ pg_bind_textdomain_codeset(const char *domainname) int new_msgenc; #ifndef WIN32 - const char *ctype = setlocale(LC_CTYPE, NULL); + const char *ctype = SETLOCALE(LC_CTYPE, NULL); if (pg_strcasecmp(ctype, "C") == 0 || pg_strcasecmp(ctype, "POSIX") == 0) #endif diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c index 4e613326056..7d13a4ddfae 100644 --- a/src/bin/initdb/initdb.c +++ b/src/bin/initdb/initdb.c @@ -81,6 +81,7 @@ #include "getopt_long.h" #include "mb/pg_wchar.h" #include "miscadmin.h" +#include "common/mdb_locale.h" /* Ideally this would be in a .h file, but it hardly seems worth the trouble */ @@ -371,7 +372,8 @@ save_global_locale(int category) if (!save) pg_fatal("out of memory"); #else - save = setlocale(category, NULL); + save = SETLOCALE(LC_TIME, NULL); + if (!save) pg_fatal("setlocale() failed"); save = pg_strdup(save); @@ -389,7 +391,8 @@ restore_global_locale(int category, save_locale_t save) if (!_wsetlocale(category, save)) pg_fatal("failed to restore old locale"); #else - if (!setlocale(category, save)) + save = pg_strdup(save); + if (!SETLOCALE(category, save)) pg_fatal("failed to restore old locale \"%s\"", save); #endif free(save); @@ -2137,7 +2140,7 @@ locale_date_order(const char *locale) save = save_global_locale(LC_TIME); - setlocale(LC_TIME, locale); + SETLOCALE(LC_TIME, locale); memset(&testtime, 0, sizeof(testtime)); testtime.tm_mday = 22; @@ -2200,7 +2203,7 @@ check_locale_name(int category, const char *locale, char **canonname) locale = ""; /* set the locale with setlocale, to see if it accepts it. */ - res = setlocale(category, locale); + res = SETLOCALE(category, locale); /* save canonical name if requested. */ if (res && canonname) diff --git a/src/bin/pg_upgrade/check.c b/src/bin/pg_upgrade/check.c index 084415feadc..be2fbbc3720 100644 --- a/src/bin/pg_upgrade/check.c +++ b/src/bin/pg_upgrade/check.c @@ -15,6 +15,8 @@ #include "fe_utils/string_utils.h" #include "mb/pg_wchar.h" #include "pg_upgrade.h" +#include "common/mdb_locale.h" + static void check_new_cluster_is_empty(void); static void check_is_install_user(ClusterInfo *cluster); diff --git a/src/common/exec.c b/src/common/exec.c index 0bee19c1e53..88583d9bc17 100644 --- a/src/common/exec.c +++ b/src/common/exec.c @@ -33,6 +33,8 @@ #include #include #include +#include "common/mdb_locale.h" + #ifdef EXEC_BACKEND #if defined(HAVE_SYS_PERSONALITY_H) @@ -453,7 +455,7 @@ set_pglocale_pgservice(const char *argv0, const char *app) /* don't set LC_ALL in the backend */ if (strcmp(app, PG_TEXTDOMAIN("postgres")) != 0) { - setlocale(LC_ALL, ""); + SETLOCALE(LC_ALL, ""); /* * One could make a case for reproducing here PostmasterMain()'s test diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index bac69bcf687..0e31315a066 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -12155,7 +12155,6 @@ proname => 'brin_minmax_multi_summary_send', provolatile => 's', prorettype => 'bytea', proargtypes => 'pg_brin_minmax_multi_summary', prosrc => 'brin_minmax_multi_summary_send' }, - { oid => '6291', descr => 'arbitrary value from among input values', proname => 'any_value', prokind => 'a', proisstrict => 'f', prorettype => 'anyelement', proargtypes => 'anyelement', @@ -12163,7 +12162,6 @@ { oid => '6292', descr => 'aggregate transition function', proname => 'any_value_transfn', prorettype => 'anyelement', proargtypes => 'anyelement anyelement', prosrc => 'any_value_transfn' }, - { oid => '6321', descr => 'list of available WAL summary files', proname => 'pg_available_wal_summaries', prorows => '100', proretset => 't', provolatile => 'v', prorettype => 'record', proargtypes => '', @@ -12231,4 +12229,8 @@ proargtypes => 'regclass', proargnames => '{relation}', prosrc => 'pg_clear_relation_stats' }, + + { oid => '16383', descr => 'contains', + proname => 'mdb_locale_enabled', prorettype => 'bool', + proargtypes => '', prosrc => 'mdb_locale_enabled' }, ] diff --git a/src/include/common/mdb_locale.h b/src/include/common/mdb_locale.h new file mode 100644 index 00000000000..61290b2d938 --- /dev/null +++ b/src/include/common/mdb_locale.h @@ -0,0 +1,24 @@ +/*------------------------------------------------------------------------- + * + * locale_mdb.h + * Generic headers for custom MDB-locales patch. + * + * IDENTIFICATION + * src/include/common/mdb_locale.h + * + *------------------------------------------------------------------------- + */ + +#ifndef PG_MDB_LOCALE_H +#define PG_MDB_LOCALE_H + +#ifdef USE_MDBLOCALES +#include +#define SETLOCALE(category, locale) mdb_setlocale(category, locale) +#define NEWLOCALE(category, locale, base) mdb_newlocale(category, locale, base) +#else +#define SETLOCALE(category, locale) setlocale(category, locale) +#define NEWLOCALE(category, locale, base) newlocale(category, locale, base) +#endif + +#endif /* PG_MDB_LOCALE_H */ \ No newline at end of file diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 6f7707f78cb..e0932e4cde1 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -645,6 +645,9 @@ /* A string containing the version number, platform, and C compiler */ #undef PG_VERSION_STR +/* Use mdb locales or not */ +#undef USE_MDBLOCALES + /* Define to 1 to allow profiling output to be saved separately for each process. */ #undef PROFILE_PID_DIR diff --git a/src/interfaces/ecpg/ecpglib/connect.c b/src/interfaces/ecpg/ecpglib/connect.c index b912441f12e..3bfb0741bfc 100644 --- a/src/interfaces/ecpg/ecpglib/connect.c +++ b/src/interfaces/ecpg/ecpglib/connect.c @@ -9,6 +9,7 @@ #include "ecpglib_extern.h" #include "ecpgtype.h" #include "sqlca.h" +#include "common/mdb_locale.h" #ifdef HAVE_USELOCALE locale_t ecpg_clocale = (locale_t) 0; @@ -495,7 +496,7 @@ ECPGconnect(int lineno, int c, const char *name, const char *user, const char *p #ifdef HAVE_USELOCALE if (!ecpg_clocale) { - ecpg_clocale = newlocale(LC_NUMERIC_MASK, "C", (locale_t) 0); + ecpg_clocale = NEWLOCALE(LC_NUMERIC_MASK, "C", (locale_t) 0); if (!ecpg_clocale) { pthread_mutex_unlock(&connections_mutex); diff --git a/src/interfaces/ecpg/ecpglib/descriptor.c b/src/interfaces/ecpg/ecpglib/descriptor.c index ad279e245c4..ba04da3f763 100644 --- a/src/interfaces/ecpg/ecpglib/descriptor.c +++ b/src/interfaces/ecpg/ecpglib/descriptor.c @@ -15,6 +15,8 @@ #include "sql3types.h" #include "sqlca.h" #include "sqlda.h" +#include "common/mdb_locale.h" + static void descriptor_free(struct descriptor *desc); @@ -493,8 +495,8 @@ ECPGget_desc(int lineno, const char *desc_name, int index,...) #ifdef HAVE__CONFIGTHREADLOCALE stmt.oldthreadlocale = _configthreadlocale(_ENABLE_PER_THREAD_LOCALE); #endif - stmt.oldlocale = ecpg_strdup(setlocale(LC_NUMERIC, NULL), lineno); - setlocale(LC_NUMERIC, "C"); + stmt.oldlocale = ecpg_strdup(SETLOCALE(LC_NUMERIC, NULL), lineno); + SETLOCALE(LC_NUMERIC, "C"); #endif /* desperate try to guess something sensible */ @@ -507,7 +509,7 @@ ECPGget_desc(int lineno, const char *desc_name, int index,...) #else if (stmt.oldlocale) { - setlocale(LC_NUMERIC, stmt.oldlocale); + SETLOCALE(LC_NUMERIC, stmt.oldlocale); ecpg_free(stmt.oldlocale); } #ifdef HAVE__CONFIGTHREADLOCALE diff --git a/src/interfaces/ecpg/ecpglib/execute.c b/src/interfaces/ecpg/ecpglib/execute.c index 04d0b40c537..9a203aec410 100644 --- a/src/interfaces/ecpg/ecpglib/execute.c +++ b/src/interfaces/ecpg/ecpglib/execute.c @@ -31,6 +31,7 @@ #include "sqlca.h" #include "sqlda-compat.h" #include "sqlda-native.h" +#include "common/mdb_locale.h" /* * This function returns a newly malloced string that has ' and \ @@ -1998,13 +1999,13 @@ ecpg_do_prologue(int lineno, const int compat, const int force_indicator, #ifdef HAVE__CONFIGTHREADLOCALE stmt->oldthreadlocale = _configthreadlocale(_ENABLE_PER_THREAD_LOCALE); #endif - stmt->oldlocale = ecpg_strdup(setlocale(LC_NUMERIC, NULL), lineno); + stmt->oldlocale = ecpg_strdup(SETLOCALE(LC_NUMERIC, NULL), lineno); if (stmt->oldlocale == NULL) { ecpg_do_epilogue(stmt); return false; } - setlocale(LC_NUMERIC, "C"); + SETLOCALE(LC_NUMERIC, "C"); #endif /* @@ -2218,7 +2219,7 @@ ecpg_do_epilogue(struct statement *stmt) uselocale(stmt->oldlocale); #else if (stmt->oldlocale) - setlocale(LC_NUMERIC, stmt->oldlocale); + SETLOCALE(LC_NUMERIC, stmt->oldlocale); #ifdef HAVE__CONFIGTHREADLOCALE /* diff --git a/src/interfaces/libpq/Makefile b/src/interfaces/libpq/Makefile index e4206ac8540..3b584bec144 100644 --- a/src/interfaces/libpq/Makefile +++ b/src/interfaces/libpq/Makefile @@ -79,7 +79,7 @@ endif # that are built correctly for use in a shlib. SHLIB_LINK_INTERNAL = -lpgcommon_shlib -lpgport_shlib ifneq ($(PORTNAME), win32) -SHLIB_LINK += $(filter -lcrypt -ldes -lcom_err -lcrypto -lk5crypto -lkrb5 -lgssapi_krb5 -lgss -lgssapi -lssl -lsocket -lnsl -lresolv -lintl -lm, $(LIBS)) $(LDAP_LIBS_FE) $(PTHREAD_LIBS) +SHLIB_LINK += $(filter -lcrypt -ldes -lcom_err -lcrypto -lk5crypto -lkrb5 -lgssapi_krb5 -lgss -lgssapi -lssl -lsocket -lnsl -lresolv -lintl -lm -lmdblocales, $(LIBS)) $(LDAP_LIBS_FE) $(PTHREAD_LIBS) else SHLIB_LINK += $(filter -lcrypt -ldes -lcom_err -lcrypto -lk5crypto -lkrb5 -lgssapi32 -lssl -lsocket -lnsl -lresolv -lintl -lm $(PTHREAD_LIBS), $(LIBS)) $(LDAP_LIBS_FE) endif diff --git a/src/pl/plperl/plperl.c b/src/pl/plperl/plperl.c index d68ad7be345..a857c1b3ebe 100644 --- a/src/pl/plperl/plperl.c +++ b/src/pl/plperl/plperl.c @@ -37,6 +37,7 @@ #include "utils/rel.h" #include "utils/syscache.h" #include "utils/typcache.h" +#include "common/mdb_locale.h" /* define our text domain for translations */ #undef TEXTDOMAIN @@ -740,15 +741,15 @@ plperl_init_interp(void) *save_numeric, *save_time; - loc = setlocale(LC_COLLATE, NULL); + loc = SETLOCALE(LC_COLLATE, NULL); save_collate = loc ? pstrdup(loc) : NULL; - loc = setlocale(LC_CTYPE, NULL); + loc = SETLOCALE(LC_CTYPE, NULL); save_ctype = loc ? pstrdup(loc) : NULL; - loc = setlocale(LC_MONETARY, NULL); + loc = SETLOCALE(LC_MONETARY, NULL); save_monetary = loc ? pstrdup(loc) : NULL; - loc = setlocale(LC_NUMERIC, NULL); + loc = SETLOCALE(LC_NUMERIC, NULL); save_numeric = loc ? pstrdup(loc) : NULL; - loc = setlocale(LC_TIME, NULL); + loc = SETLOCALE(LC_TIME, NULL); save_time = loc ? pstrdup(loc) : NULL; #define PLPERL_RESTORE_LOCALE(name, saved) \ @@ -4181,7 +4182,7 @@ static char * setlocale_perl(int category, char *locale) { dTHX; - char *RETVAL = setlocale(category, locale); + char *RETVAL = SETLOCALE(category, locale); if (RETVAL) { @@ -4196,7 +4197,7 @@ setlocale_perl(int category, char *locale) #ifdef LC_ALL if (category == LC_ALL) - newctype = setlocale(LC_CTYPE, NULL); + newctype = SETLOCALE(LC_CTYPE, NULL); else #endif newctype = RETVAL; @@ -4214,7 +4215,7 @@ setlocale_perl(int category, char *locale) #ifdef LC_ALL if (category == LC_ALL) - newcoll = setlocale(LC_COLLATE, NULL); + newcoll = SETLOCALE(LC_COLLATE, NULL); else #endif newcoll = RETVAL; @@ -4233,7 +4234,7 @@ setlocale_perl(int category, char *locale) #ifdef LC_ALL if (category == LC_ALL) - newnum = setlocale(LC_NUMERIC, NULL); + newnum = SETLOCALE(LC_NUMERIC, NULL); else #endif newnum = RETVAL; diff --git a/src/port/chklocale.c b/src/port/chklocale.c index 8cb81c8640e..49c1ff138db 100644 --- a/src/port/chklocale.c +++ b/src/port/chklocale.c @@ -18,6 +18,8 @@ #else #include "postgres_fe.h" #endif +#include "common/mdb_locale.h" + #ifdef HAVE_LANGINFO_H #include @@ -319,7 +321,7 @@ pg_get_encoding_from_locale(const char *ctype, bool write_message) pg_strcasecmp(ctype, "POSIX") == 0) return PG_SQL_ASCII; - save = setlocale(LC_CTYPE, NULL); + save = SETLOCALE(LC_CTYPE, NULL); if (!save) return -1; /* setlocale() broken? */ /* must copy result, or it might change after setlocale */ @@ -327,7 +329,7 @@ pg_get_encoding_from_locale(const char *ctype, bool write_message) if (!save) return -1; /* out of memory; unlikely */ - name = setlocale(LC_CTYPE, ctype); + name = SETLOCALE(LC_CTYPE, ctype); if (!name) { free(save); @@ -342,13 +344,13 @@ pg_get_encoding_from_locale(const char *ctype, bool write_message) sys = win32_langinfo(name); #endif - setlocale(LC_CTYPE, save); + SETLOCALE(LC_CTYPE, save); free(save); } else { /* much easier... */ - ctype = setlocale(LC_CTYPE, NULL); + ctype = SETLOCALE(LC_CTYPE, NULL); if (!ctype) return -1; /* setlocale() broken? */ diff --git a/src/test/locale/test-ctype.c b/src/test/locale/test-ctype.c index a3f896c5ecb..10c2b49cb92 100644 --- a/src/test/locale/test-ctype.c +++ b/src/test/locale/test-ctype.c @@ -23,6 +23,8 @@ the author shall be liable for any damage, etc. #include #include #include +#include "common/mdb_locale.h" + char *flag(int b); void describe_char(int c); @@ -62,7 +64,7 @@ main() short c; char *cur_locale; - cur_locale = setlocale(LC_ALL, ""); + cur_locale = SETLOCALE(LC_ALL, ""); if (cur_locale) fprintf(stderr, "Successfully set locale to \"%s\"\n", cur_locale); else diff --git a/src/test/regress/expected/misc.out b/src/test/regress/expected/misc.out index 6e816c57f1f..d2cbc8f87f4 100644 --- a/src/test/regress/expected/misc.out +++ b/src/test/regress/expected/misc.out @@ -396,3 +396,10 @@ SELECT *, (equipment(CAST((h.*) AS hobbies_r))).name FROM hobbies_r h; -- -- rewrite rules -- +--- mdb-related +SELECT mdb_locale_enabled(); + mdb_locale_enabled +-------------------- + t +(1 row) + diff --git a/src/test/regress/sql/misc.sql b/src/test/regress/sql/misc.sql index 165a2e175fb..597186ffe4b 100644 --- a/src/test/regress/sql/misc.sql +++ b/src/test/regress/sql/misc.sql @@ -273,3 +273,8 @@ SELECT *, (equipment(CAST((h.*) AS hobbies_r))).name FROM hobbies_r h; -- -- rewrite rules -- + +--- mdb-related + +SELECT mdb_locale_enabled(); + From 24fde0512cb201df3d3e0e7e31ba7080631d5f0c Mon Sep 17 00:00:00 2001 From: reshke kirill Date: Thu, 11 May 2023 08:36:59 +0000 Subject: [PATCH 28/54] MDB-23247: startup param for auth passthrough under unpriviledged user MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Accept new startup param _pq_.service_auth_role for service auth under any user, which is niether superuser nor have some dangerous system role priv Add tap-test for mdb service role auth 👍👌😉 Fix tests after rebase contrib tests 💅️️💅️️💅️️ now works MDB-23247: debug ouput for testing purposes lowered to DEBUG5 elog level Skip caching routines for pre-startup logic (SCRAM -service auth role) --- contrib/seg/Makefile | 4 +- src/backend/libpq/auth.c | 26 +++++++- src/backend/tcop/backend_startup.c | 21 ++++--- src/backend/utils/adt/acl.c | 70 ++++++++++++++++----- src/bin/psql/t/100_mdb.pl | 98 +++++++++++++++++++++++++++++ src/include/libpq/libpq-be.h | 4 ++ src/include/utils/acl.h | 6 +- src/interfaces/libpq/fe-connect.c | 9 +++ src/interfaces/libpq/fe-protocol3.c | 4 ++ src/interfaces/libpq/libpq-int.h | 1 + 10 files changed, 218 insertions(+), 25 deletions(-) create mode 100644 src/bin/psql/t/100_mdb.pl diff --git a/contrib/seg/Makefile b/contrib/seg/Makefile index b408f4049cb..95673eec524 100644 --- a/contrib/seg/Makefile +++ b/contrib/seg/Makefile @@ -13,8 +13,10 @@ DATA = seg--1.1.sql seg--1.1--1.2.sql seg--1.2--1.3.sql seg--1.3--1.4.sql \ PGFILEDESC = "seg - line segment data type" HEADERS = segdata.h +# MDB-23261 diasble security test, because we do not extension creation via sql for now +#REGRESS = security seg -REGRESS = security seg partition +REGRESS = seg partition EXTRA_CLEAN = segparse.h segparse.c segscan.c diff --git a/src/backend/libpq/auth.c b/src/backend/libpq/auth.c index e0576f801d8..701c35f7b12 100644 --- a/src/backend/libpq/auth.c +++ b/src/backend/libpq/auth.c @@ -38,6 +38,8 @@ #include "replication/walsender.h" #include "storage/ipc.h" #include "utils/memutils.h" +#include "utils/timestamp.h" +#include "utils/acl.h" /*---------------------------------------------------------------- * Global authentication functions @@ -832,12 +834,32 @@ CheckPWChallengeAuth(Port *port, const char **logdetail) int auth_result; char *shadow_pass; PasswordType pwtype; + Oid mdb_service_authoid; + Oid useroid; + Oid service_auth_roleoid; Assert(port->hba->auth_method == uaSCRAM || port->hba->auth_method == uaMD5); - /* First look up the user's password. */ - shadow_pass = get_role_password(port->user_name, logdetail); + + + if (port->service_auth_role) { + mdb_service_authoid = get_role_oid("mdb_service_auth", true); + service_auth_roleoid = get_role_oid(port->service_auth_role, true); + useroid = get_role_oid(port->user_name, true); + + /* MDB-23247: check that given role name has priviledge for auth - passthrough*/ + if (!is_member_of_role(service_auth_roleoid, mdb_service_authoid) || has_privs_of_unwanted_system_role_prestartup(useroid)) + ereport(FATAL, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("invalid auth request for role %s", + port->service_auth_role))); + + shadow_pass = get_role_password(port->service_auth_role, logdetail); + } else { + /* First look up the user's password. */ + shadow_pass = get_role_password(port->user_name, logdetail); + } /* * If the user does not exist, or has no password or it's expired, we diff --git a/src/backend/tcop/backend_startup.c b/src/backend/tcop/backend_startup.c index ce1275bccfe..a706c0d8f65 100644 --- a/src/backend/tcop/backend_startup.c +++ b/src/backend/tcop/backend_startup.c @@ -715,6 +715,8 @@ ProcessStartupPacket(Port *port, bool ssl_done, bool gss_done) break; /* missing value, will complain below */ valptr = buf + valoffset; + elog(DEBUG5, "startup got %s %s", nameptr, valptr); + if (strcmp(nameptr, "database") == 0) port->database_name = pstrdup(valptr); else if (strcmp(nameptr, "user") == 0) @@ -745,13 +747,18 @@ ProcessStartupPacket(Port *port, bool ssl_done, bool gss_done) } else if (strncmp(nameptr, "_pq_.", 5) == 0) { - /* - * Any option beginning with _pq_. is reserved for use as a - * protocol-level option, but at present no such options are - * defined. - */ - unrecognized_protocol_options = - lappend(unrecognized_protocol_options, pstrdup(nameptr)); + /* MDB-23247: parse service auth role from startup options */ + if (strcmp(nameptr, "_pq_.service_auth_role") == 0) { + port->service_auth_role = pstrdup(valptr); + } else { + /* + * Any option beginning with _pq_. is reserved for use as a + * protocol-level option, but at present no such options are + * defined. + */ + unrecognized_protocol_options = + lappend(unrecognized_protocol_options, pstrdup(nameptr)); + } } else { diff --git a/src/backend/utils/adt/acl.c b/src/backend/utils/adt/acl.c index c60d22a1ba3..8d71e5b9a91 100644 --- a/src/backend/utils/adt/acl.c +++ b/src/backend/utils/adt/acl.c @@ -129,8 +129,6 @@ static AclResult pg_role_aclcheck(Oid role_oid, Oid roleid, AclMode mode); static void RoleMembershipCacheCallback(Datum arg, int cacheid, uint32 hashvalue); -static bool has_privs_of_unwanted_system_role(Oid role); - /* * Test whether an identifier char can be left unquoted in ACLs. @@ -5019,7 +5017,7 @@ roles_list_append(List *roles_list, bloom_filter **bf, Oid role) */ static List * roles_is_member_of(Oid roleid, enum RoleRecurseType type, - Oid admin_of, Oid *admin_role) + Oid admin_of, Oid *admin_role, bool no_cache) { Oid dba; List *roles_list; @@ -5036,7 +5034,6 @@ roles_is_member_of(Oid roleid, enum RoleRecurseType type, if (cached_role[type] == roleid && !OidIsValid(admin_of) && OidIsValid(cached_role[type])) return cached_roles[type]; - /* * Role expansion happens in a non-database backend when guc.c checks * ROLE_PG_READ_ALL_SETTINGS for a physical walsender SHOW command. In @@ -5134,7 +5131,10 @@ roles_is_member_of(Oid roleid, enum RoleRecurseType type, cached_role[type] = InvalidOid; /* just paranoia */ list_free(cached_roles[type]); cached_roles[type] = new_cached_roles; - cached_role[type] = roleid; + + if (!no_cache) { + cached_role[type] = roleid; + } /* And now we can return the answer */ return cached_roles[type]; @@ -5170,17 +5170,39 @@ has_privs_of_role_strict(Oid member, Oid role) * multi-level recursion, then see if target role is any one of them. */ return list_member_oid(roles_is_member_of(member, ROLERECURSE_PRIVS, - InvalidOid, NULL), + InvalidOid, NULL, false), + role); +} + + +static bool +has_privs_of_role_strict_no_cache(Oid member, Oid role) +{ + /* Fast path for simple case */ + if (member == role) + return true; + + /* Superusers have every privilege, so are part of every role */ + if (superuser_arg(member)) + return true; + + /* + * Find all the roles that member has the privileges of, including + * multi-level recursion, then see if target role is any one of them. + */ + return list_member_oid(roles_is_member_of(member, ROLERECURSE_PRIVS, + InvalidOid, NULL, true), role); } + /* * Check that role is either one of "dangerous" system role * or has "strict" (not through mdb_admin or mdb_superuser) * privs of this role */ -static bool +bool has_privs_of_unwanted_system_role(Oid role) { if (has_privs_of_role_strict(role, ROLE_PG_READ_SERVER_FILES)) { return true; @@ -5201,6 +5223,26 @@ has_privs_of_unwanted_system_role(Oid role) { return false; } +bool has_privs_of_unwanted_system_role_prestartup(Oid role) { + if (has_privs_of_role_strict_no_cache(role, ROLE_PG_READ_SERVER_FILES)) { + return true; + } + if (has_privs_of_role_strict_no_cache(role, ROLE_PG_WRITE_SERVER_FILES)) { + return true; + } + if (has_privs_of_role_strict_no_cache(role, ROLE_PG_EXECUTE_SERVER_PROGRAM)) { + return true; + } + if (has_privs_of_role_strict_no_cache(role, ROLE_PG_READ_ALL_DATA)) { + return true; + } + if (has_privs_of_role_strict_no_cache(role, ROLE_PG_WRITE_ALL_DATA)) { + return true; + } + + return false; +} + bool has_privs_of_role(Oid member, Oid role) { @@ -5236,7 +5278,7 @@ has_privs_of_role(Oid member, Oid role) * multi-level recursion, then see if target role is any one of them. */ return list_member_oid(roles_is_member_of(member, ROLERECURSE_PRIVS, - InvalidOid, NULL), + InvalidOid, NULL, false), role); } @@ -5313,7 +5355,7 @@ member_can_set_role(Oid member, Oid role) * multi-level recursion, then see if target role is any one of them. */ return list_member_oid(roles_is_member_of(member, ROLERECURSE_SETROLE, - InvalidOid, NULL), + InvalidOid, NULL, false), role); } @@ -5426,7 +5468,7 @@ is_member_of_role(Oid member, Oid role) * recursion, then see if target role is any one of them. */ return list_member_oid(roles_is_member_of(member, ROLERECURSE_MEMBERS, - InvalidOid, NULL), + InvalidOid, NULL, false), role); } @@ -5450,7 +5492,7 @@ is_member_of_role_nosuper(Oid member, Oid role) * recursion, then see if target role is any one of them. */ return list_member_oid(roles_is_member_of(member, ROLERECURSE_MEMBERS, - InvalidOid, NULL), + InvalidOid, NULL, false), role); } @@ -5472,7 +5514,7 @@ is_admin_of_role(Oid member, Oid role) if (member == role) return false; - (void) roles_is_member_of(member, ROLERECURSE_MEMBERS, role, &admin_role); + (void) roles_is_member_of(member, ROLERECURSE_MEMBERS, role, &admin_role, false); return OidIsValid(admin_role); } @@ -5494,7 +5536,7 @@ select_best_admin(Oid member, Oid role) if (member == role) return InvalidOid; - (void) roles_is_member_of(member, ROLERECURSE_PRIVS, role, &admin_role); + (void) roles_is_member_of(member, ROLERECURSE_PRIVS, role, &admin_role, false); return admin_role; } @@ -5581,7 +5623,7 @@ select_best_grantor(Oid roleId, AclMode privileges, * doesn't query any role memberships. */ roles_list = roles_is_member_of(roleId, ROLERECURSE_PRIVS, - InvalidOid, NULL); + InvalidOid, NULL, false); /* initialize candidate result as default */ *grantorId = roleId; *grantOptions = ACL_NO_RIGHTS; diff --git a/src/bin/psql/t/100_mdb.pl b/src/bin/psql/t/100_mdb.pl new file mode 100644 index 00000000000..d5b0342354c --- /dev/null +++ b/src/bin/psql/t/100_mdb.pl @@ -0,0 +1,98 @@ + +# Copyright (c) 2021-2023, PostgreSQL Global Development Group + +use strict; +use warnings; + +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +# Initialize primary node. Force UTF-8 encoding, so that we can use non-ASCII +# characters in the passwords below. +my $node = PostgreSQL::Test::Cluster->new('primary'); +$node->init(extra => [ '--locale=C', '--encoding=UTF8' ]); +$node->start; + + +# Create test roles. +$node->safe_psql( + 'postgres', + "SET password_encryption='scram-sha-256'; +SET client_encoding='utf8'; +CREATE USER mdbsar_test_role LOGIN PASSWORD 'test'; + +CREATE ROLE mdb_service_auth LOGIN PASSWORD 'serv'; +CREATE ROLE sup LOGIN SUPERUSER PASSWORD '123'; +"); + + +# Delete pg_hba.conf from the given node, add a new entry to it +# and then execute a reload to refresh it. +sub reset_pg_hba +{ + my $node = shift; + my $hba_method = shift; + + unlink($node->data_dir . '/pg_hba.conf'); + $node->append_conf('pg_hba.conf', "local all all $hba_method"); + $node->reload; + return; +} + +# Require password from now on. +reset_pg_hba($node, 'scram-sha-256'); + + + +# Test access for a single role, useful to wrap all tests into one. +sub test_login +{ + local $Test::Builder::Level = $Test::Builder::Level + 1; + + my $node = shift; + my $role = shift; + my $password = shift; + my $expected_res = shift; + my $add_serv_role = shift; + my $status_string = 'failed'; + + $status_string = 'success' if ($expected_res eq 0); + + my $connstr = "user=$role"; + my $testname = + "authentication $status_string for role $role with password $password"; + + $ENV{"PGPASSWORD"} = $password; + if ($add_serv_role eq 1) + { + $ENV{"PGSERVICEAUTHROLE"} = 'mdb_service_auth'; + } + + if ($expected_res eq 0) + { + $node->connect_ok($connstr, $testname); + } + else + { + # No checks of the error message, only the status code. + $node->connect_fails($connstr, $testname); + } +} + + +test_login($node, 'mdb_service_auth', "serv", 0, 0); +test_login($node, 'mdbsar_test_role', "serv", 2, 0); +test_login($node, 'mdbsar_test_role', "test", 0, 0); + +test_login($node, 'sup', "123", 0, 0); + +test_login($node, 'mdb_service_auth', "serv", 0, 1); +test_login($node, 'mdbsar_test_role', "serv", 0, 1); +test_login($node, 'mdbsar_test_role', "test", 2, 1); + +test_login($node, 'sup', "serv", 2, 1); + +ok(1); +done_testing(); + diff --git a/src/include/libpq/libpq-be.h b/src/include/libpq/libpq-be.h index 6a1a2274c55..0e44c490a06 100644 --- a/src/include/libpq/libpq-be.h +++ b/src/include/libpq/libpq-be.h @@ -150,6 +150,10 @@ typedef struct Port */ char *database_name; char *user_name; + /* + * MDB-23247: service role name to perform auth - passthrough + */ + char *service_auth_role; char *cmdline_options; List *guc_options; diff --git a/src/include/utils/acl.h b/src/include/utils/acl.h index 1887581b6d9..38011873bf3 100644 --- a/src/include/utils/acl.h +++ b/src/include/utils/acl.h @@ -212,8 +212,11 @@ extern int aclmembers(const Acl *acl, Oid **roleids); extern bool has_privs_of_role(Oid member, Oid role); extern bool member_can_set_role(Oid member, Oid role); extern void check_can_set_role(Oid member, Oid role); -extern bool has_privs_of_role_strict(Oid member, Oid role); extern bool is_member_of_role(Oid member, Oid role); + +// -- mdb patch +extern bool has_privs_of_unwanted_system_role_prestartup(Oid role); +// -- mdb patch end extern bool is_member_of_role_nosuper(Oid member, Oid role); extern bool is_admin_of_role(Oid member, Oid role); @@ -228,6 +231,7 @@ extern bool mdb_admin_is_member_of_role(Oid member, Oid role); extern Oid select_best_admin(Oid member, Oid role); extern Oid get_role_oid(const char *rolname, bool missing_ok); extern Oid get_role_oid_or_public(const char *rolname); +extern bool has_privs_of_unwanted_system_role(Oid role); extern Oid get_rolespec_oid(const RoleSpec *role, bool missing_ok); extern void check_rolespec_name(const RoleSpec *role, const char *detail_msg); extern HeapTuple get_rolespec_tuple(const RoleSpec *role); diff --git a/src/interfaces/libpq/fe-connect.c b/src/interfaces/libpq/fe-connect.c index 5d390c1d8f3..f88ba134fe9 100644 --- a/src/interfaces/libpq/fe-connect.c +++ b/src/interfaces/libpq/fe-connect.c @@ -365,6 +365,12 @@ static const internalPQconninfoOption PQconninfoOptions[] = { "Load-Balance-Hosts", "", 8, /* sizeof("disable") = 8 */ offsetof(struct pg_conn, load_balance_hosts)}, + /* MDB-23247: option for service log-in */ + {"_pq_.service_auth_role", "PGSERVICEAUTHROLE", + "", NULL, + "_pg__service_auth_role", "", 20, + offsetof(struct pg_conn, service_auth_role)}, + /* Terminating entry --- MUST BE LAST */ {NULL, NULL, NULL, NULL, NULL, NULL, 0} @@ -4703,6 +4709,9 @@ freePGconn(PGconn *conn) free(conn->rowBuf); free(conn->target_session_attrs); free(conn->load_balance_hosts); + if (conn->service_auth_role) { + free(conn->service_auth_role); + } termPQExpBuffer(&conn->errorMessage); termPQExpBuffer(&conn->workBuffer); diff --git a/src/interfaces/libpq/fe-protocol3.c b/src/interfaces/libpq/fe-protocol3.c index a51652b4306..f37fe592680 100644 --- a/src/interfaces/libpq/fe-protocol3.c +++ b/src/interfaces/libpq/fe-protocol3.c @@ -2358,6 +2358,10 @@ build_startup_packet(const PGconn *conn, char *packet, } } + /* MDB-23247: add service auth role to startup options */ + if (conn->service_auth_role && conn->service_auth_role[0]) + ADD_STARTUP_OPTION("_pq_.service_auth_role", conn->service_auth_role); + /* Add trailing terminator */ if (packet) packet[packet_len] = '\0'; diff --git a/src/interfaces/libpq/libpq-int.h b/src/interfaces/libpq/libpq-int.h index e7fa4e8008f..d99c2c9c211 100644 --- a/src/interfaces/libpq/libpq-int.h +++ b/src/interfaces/libpq/libpq-int.h @@ -420,6 +420,7 @@ struct pg_conn bool cancelRequest; /* true if this connection is used to send a * cancel request, instead of being a normal * connection that's used for queries */ + char *service_auth_role; /* MDB-23247: option for service log-in */ /* Optional file to write trace info to */ FILE *Pfdebug; From 1fa61788e4af54545416882f9811f27f017f50cd Mon Sep 17 00:00:00 2001 From: reshke kirill Date: Wed, 5 Jul 2023 07:06:41 +0000 Subject: [PATCH 29/54] Add mdb changelog Update mdb-patched.md Update mdb-pacthes.md --- MDB-PATCHES.md | 90 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 MDB-PATCHES.md diff --git a/MDB-PATCHES.md b/MDB-PATCHES.md new file mode 100644 index 00000000000..a7d1efcbefc --- /dev/null +++ b/MDB-PATCHES.md @@ -0,0 +1,90 @@ + +15: + +7dfdf3a55ed: mdb_replication role patch +f6a3406fed8: Disallow cancelation of syncronous commit V1 +4289037edd2: Extend multixact SLRU +d6842b7f65c: Allow mdb_admin to create LEAKPROOF functions +146a82b3a75: mdb admin sets session replication role +68312eae50b: [MDB-16648]: Allow mdb admin to kill specific superuser queries +ab8f5243195: provide [mdb -postgresql] restict grant roles in YC[MDB-16990] +8ccf1aa7c51: Allow mdb admin to tranfers ownership on non-superuser objects regressioon tests for mdb admin functionality[MDB-16988] +bf4cad5ecdd: MDB-17910: check MDB reserved application name fix +64fed445a14: MDB-16955 : disallow to kill repl mon in cloud +afeb1eb4c2f: Fix mdb_replication role +5852300fb1d: pg_replication_slot_advance fix +c52bd070b56: Fix compilation errors + +0db90bbcbdd: Demonstrate and fix lock of all SQL queries by pg_stat_statements +530019f966d: MDB-21297: forbit usage of COPY TO PROGRAMM and COPY FROM PROGRAMM to non-superuser +8c860bf4d66: Reimplement mdb-admin, refactor mdb_admin check and usages. + +3a89cc36c74: Implement mdb-locales patch +dc7d503498b: Add mdb locales patch, restore COPY from/to files, enable regress. +96c30d707a7: Role mdb_superuser: feature and regress testsing +2d5f40ce3c9: Refactor optional setlocale, fix minor issues +41f04495a89: Update dependencies: bump libmdblocales, add mdb-locales +adc0b21d39f: Allow mdb_superuser to have power of pg_database_owner +ac90e1819fa: MDB-23247: startup param for auth passthrough under unpriviledged user +2bf6f042542: Add tap-test for mdb service role auth 👍👌😉 +9750b4efc44: Use fadvise to prefetch WAL in xlogrecovery +25f12802528: Fix tests after rebasecontrib tests 💅️️💅️️💅️️ now works +746dd65f557: MDB-23247: debug ouput for testing purposes lowered to DEBUG5 elog level + + + + + +16: + +/* misc */ + +/* on branch mdb-16 cherry-picked 'as is' */ +f6a3406fed8 -> 1effb23478e: Disallow cancelation of syncronous commit V1 +4289037edd2 -> b542d608604: Extend multixact SLRU + + + +/* mdb - admin + mdb_replication */ +7dfdf3a55ed: mdb_replication role patch + + +d6842b7f65c: Allow mdb_admin to create LEAKPROOF functions +146a82b3a75: mdb admin sets session replication role +68312eae50b: [MDB-16648]: Allow mdb admin to kill specific superuser queries +8ccf1aa7c51: Allow mdb admin to tranfers ownership on non-superuser objects regressioon tests for mdb admin functionality[MDB-16988] +8c860bf4d66: Reimplement mdb-admin, refactor mdb_admin check and usages. + +/* sqashed to */ +52435055d7b: Mdb-admin patch and regression tests +/*******/ + +/* as is */ +ab8f5243195->3fecc85426e: provide [mdb -postgresql] restict grant roles in YC[MDB-16990] + +/* pack of mdb patches */ +bf4cad5ecdd: MDB-17910: check MDB reserved application name fix +64fed445a14: MDB-16955 : disallow to kill repl mon in cloud +afeb1eb4c2f: Fix mdb_replication role +5852300fb1d: pg_replication_slot_advance fix +c52bd070b56: Fix compilation errors + +/* squashed to */ +52ea09c2d90: Pack of MDB-related patches: +/* */ + +0db90bbcbdd: Demonstrate and fix lock of all SQL queries by pg_stat_statements +530019f966d: MDB-21297: forbit usage of COPY TO PROGRAMM and COPY FROM PROGRAMM to non-superuser + +3a89cc36c74: Implement mdb-locales patch +dc7d503498b: Add mdb locales patch, restore COPY from/to files, enable regress. +96c30d707a7: Role mdb_superuser: feature and regress testsing +2d5f40ce3c9: Refactor optional setlocale, fix minor issues +41f04495a89: Update dependencies: bump libmdblocales, add mdb-locales +adc0b21d39f: Allow mdb_superuser to have power of pg_database_owner +ac90e1819fa: MDB-23247: startup param for auth passthrough under unpriviledged user +2bf6f042542: Add tap-test for mdb service role auth 👍👌😉 +9750b4efc44: Use fadvise to prefetch WAL in xlogrecovery +25f12802528: Fix tests after rebasecontrib tests 💅️️💅️️💅️️ now works +746dd65f557: MDB-23247: debug ouput for testing purposes lowered to DEBUG5 elog level + From 9dc5348fb49116eb7c912a4278926e1a48a4dad2 Mon Sep 17 00:00:00 2001 From: Kirill Reshke Date: Tue, 28 May 2024 14:18:25 +0300 Subject: [PATCH 30/54] Restrict DROP DATABASE to superuser only --- src/backend/commands/dbcommands.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/backend/commands/dbcommands.c b/src/backend/commands/dbcommands.c index f529e0ff129..5b76c2b2fce 100644 --- a/src/backend/commands/dbcommands.c +++ b/src/backend/commands/dbcommands.c @@ -1682,6 +1682,15 @@ dropdb(const char *dbname, bool missing_ok, bool force) aclcheck_error(ACLCHECK_NOT_OWNER, OBJECT_DATABASE, dbname); + /* -- MDB kostyl begin + * remove this when feature will be supported + */ + if (!superuser()) + aclcheck_error(ACLCHECK_NO_PRIV, OBJECT_DATABASE, + dbname); + + /* MDB kostyl end */ + /* DROP hook for the database being removed */ InvokeObjectDropHook(DatabaseRelationId, db_id, 0); From 303adf9e9610bef24bd67d27f47d97eb1a9c9476 Mon Sep 17 00:00:00 2001 From: Andrey Borodin Date: Mon, 8 Jul 2024 14:57:31 +0500 Subject: [PATCH 31/54] [MDB-28474] Increate readaheadchunk for XlogPageReader() We learned from the field incidents, that 128Kb is not enough. This time we are going to try to increase this value. --- src/backend/access/transam/xlogrecovery.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c index 8134f38de8c..b1ea5726a92 100644 --- a/src/backend/access/transam/xlogrecovery.c +++ b/src/backend/access/transam/xlogrecovery.c @@ -3438,6 +3438,18 @@ XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen, Assert(targetPageOff == readOff); Assert(reqLen <= readLen); +#if defined(USE_POSIX_FADVISE) && defined(POSIX_FADV_WILLNEED) + /* + * Prefetch next wal blocks to avoid page misses on next read iterations. + */ +#define RACHUNK (16*1024*1024) + if (readOff % RACHUNK == 0) { + pgstat_report_wait_start(WAIT_EVENT_WAL_PREFETCH); + posix_fadvise(readFile, readOff + RACHUNK, RACHUNK, POSIX_FADV_WILLNEED); + pgstat_report_wait_end(); + } +#endif + xlogreader->seg.ws_tli = curFileTLI; /* From 9221cfa43d97078e27403a25099bc548a0ae0e21 Mon Sep 17 00:00:00 2001 From: Jakub Wartak Date: Thu, 23 Jun 2022 08:18:26 +0000 Subject: [PATCH 32/54] Use fadvise to prefetch WAL in xlogrecovery Use fadvise in walsender remove bogus progress reporting Fix fadvise patch --- src/backend/access/transam/xlogreader.c | 10 +++++++++ src/backend/access/transam/xlogrecovery.c | 22 +++++++++---------- .../utils/activity/wait_event_names.txt | 1 + 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 7680c234d30..714e779e9b8 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -1567,6 +1567,16 @@ WALRead(XLogReaderState *state, /* Reset errno first; eases reporting non-errno-affecting errors */ errno = 0; + +#if defined(USE_POSIX_FADVISE) && defined(POSIX_FADV_WILLNEED) + /* + * Prefetch next wal blocks to avoid page misses on next read iterations. + */ +#define RACHUNK (16*1024*1024) + if (p == 0) + posix_fadvise(state->seg.ws_file, 0, RACHUNK, POSIX_FADV_WILLNEED); + +#endif readbytes = pg_pread(state->seg.ws_file, p, segbytes, (off_t) startoff); #ifndef FRONTEND diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c index b1ea5726a92..dbd93ecb4b0 100644 --- a/src/backend/access/transam/xlogrecovery.c +++ b/src/backend/access/transam/xlogrecovery.c @@ -3407,6 +3407,16 @@ XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen, readOff = targetPageOff; pgstat_report_wait_start(WAIT_EVENT_WAL_READ); + +#if defined(USE_POSIX_FADVISE) && defined(POSIX_FADV_WILLNEED) + /* + * Prefetch next wal blocks to avoid page misses on next read iterations. + */ +#define RACHUNK (16*1024*1024) + if (readOff == 0) { + posix_fadvise(readFile, 0, RACHUNK, POSIX_FADV_WILLNEED); + } +#endif r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) readOff); if (r != XLOG_BLCKSZ) { @@ -3438,18 +3448,6 @@ XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen, Assert(targetPageOff == readOff); Assert(reqLen <= readLen); -#if defined(USE_POSIX_FADVISE) && defined(POSIX_FADV_WILLNEED) - /* - * Prefetch next wal blocks to avoid page misses on next read iterations. - */ -#define RACHUNK (16*1024*1024) - if (readOff % RACHUNK == 0) { - pgstat_report_wait_start(WAIT_EVENT_WAL_PREFETCH); - posix_fadvise(readFile, readOff + RACHUNK, RACHUNK, POSIX_FADV_WILLNEED); - pgstat_report_wait_end(); - } -#endif - xlogreader->seg.ws_tli = curFileTLI; /* diff --git a/src/backend/utils/activity/wait_event_names.txt b/src/backend/utils/activity/wait_event_names.txt index 426a1c6ff64..3702559b479 100644 --- a/src/backend/utils/activity/wait_event_names.txt +++ b/src/backend/utils/activity/wait_event_names.txt @@ -262,6 +262,7 @@ WAL_COPY_WRITE "Waiting for a write when creating a new WAL segment by copying a WAL_INIT_SYNC "Waiting for a newly initialized WAL file to reach durable storage." WAL_INIT_WRITE "Waiting for a write while initializing a new WAL file." WAL_READ "Waiting for a read from a WAL file." +WAL_PREFETCH "Waiting for a WAL file prefetching." WAL_SUMMARY_READ "Waiting for a read from a WAL summary file." WAL_SUMMARY_WRITE "Waiting for a write to a WAL summary file." WAL_SYNC "Waiting for a WAL file to reach durable storage." From 31cc6319be7991d64e83547033cf1e2697495791 Mon Sep 17 00:00:00 2001 From: diphantxm Date: Mon, 9 Sep 2024 16:50:15 +0300 Subject: [PATCH 33/54] parameter max_log_size to truncate logs There is no need to log the entire query, because it may be large and take lots of space on disk. Parameter max_log_size set the maximum length for logged query. Everything beyond that length is truncated. Value 0 disables the parameter. --- src/backend/utils/error/elog.c | 8 ++++++++ src/backend/utils/misc/guc_tables.c | 11 +++++++++++ src/backend/utils/misc/postgresql.conf.sample | 2 ++ src/bin/pg_ctl/t/004_logrotate.pl | 15 +++++++++++++++ src/include/utils/elog.h | 1 + 5 files changed, 37 insertions(+) diff --git a/src/backend/utils/error/elog.c b/src/backend/utils/error/elog.c index 4f66acc5a6e..77bbd9230dc 100644 --- a/src/backend/utils/error/elog.c +++ b/src/backend/utils/error/elog.c @@ -113,6 +113,7 @@ int Log_destination = LOG_DESTINATION_STDERR; char *Log_destination_string = NULL; bool syslog_sequence_numbers = true; bool syslog_split_messages = true; +int max_log_size = 0; /* Processed form of backtrace_functions GUC */ static char *backtrace_function_list; @@ -1693,6 +1694,13 @@ EmitErrorReport(void) CHECK_STACK_DEPTH(); oldcontext = MemoryContextSwitchTo(edata->assoc_context); + if (max_log_size != 0 && debug_query_string != NULL) + { + char* str = debug_query_string; + str[pg_mbcliplen(str, strlen(str), max_log_size)] = '\0'; + debug_query_string = str; + } + /* * Reset the formatted timestamp fields before emitting any logs. This * includes all the log destinations and emit_log_hook, as the latter diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index 5d961576a95..63d28cae8e9 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -3685,6 +3685,17 @@ struct config_int ConfigureNamesInt[] = NULL, NULL, NULL }, + { + {"max_log_size", PGC_SIGHUP, LOGGING_WHAT, + gettext_noop("Sets max size of logged statement."), + NULL + }, + &max_log_size, + 5 * (1024 * 1024), + 0, INT_MAX, + NULL, NULL, NULL + }, + /* End-of-list marker */ { {NULL, 0, 0, NULL, NULL}, NULL, 0, 0, 0, NULL, NULL, NULL diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index 97c48c17076..95c086d2b4b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -620,6 +620,8 @@ # bind-parameter values to N bytes; # -1 means print in full, 0 disables #log_statement = 'none' # none, ddl, mod, all +#max_log_size = 0 # max size of logged statement_timeout + # 0 disables the feature #log_replication_commands = off #log_temp_files = -1 # log temporary files equal or larger # than the specified size in kilobytes; diff --git a/src/bin/pg_ctl/t/004_logrotate.pl b/src/bin/pg_ctl/t/004_logrotate.pl index eacca1a6523..1c1f89cb0aa 100644 --- a/src/bin/pg_ctl/t/004_logrotate.pl +++ b/src/bin/pg_ctl/t/004_logrotate.pl @@ -69,6 +69,7 @@ sub check_log_pattern # these ensure stability of test results: log_rotation_age = 0 lc_messages = 'C' +max_log_size = 32 )); $node->start(); @@ -135,6 +136,20 @@ sub check_log_pattern check_log_pattern('csvlog', $new_current_logfiles, 'syntax error', $node); check_log_pattern('jsonlog', $new_current_logfiles, 'syntax error', $node); +$node->psql('postgres', 'INSERT INTO SOME_NON_EXISTANT_TABLE VALUES (TEST)'); +for (my $attempts = 0; $attempts < $max_attempts; $attempts++) +{ + eval { + $current_logfiles = slurp_file($node->data_dir . '/current_logfiles'); + }; + last unless $@; + usleep(100_000); +} +die $@ if $@; +check_log_pattern('stderr', $current_logfiles, 'INSERT INTO SOME_NON_EXISTANT_TA(?!(BLE VALUES \(TEST\)))', $node); +check_log_pattern('csvlog', $current_logfiles, 'INSERT INTO SOME_NON_EXISTANT_TA(?!(BLE VALUES \(TEST\)))', $node); +check_log_pattern('jsonlog', $current_logfiles, 'INSERT INTO SOME_NON_EXISTANT_TA(?!(BLE VALUES \(TEST\)))', $node); + $node->stop(); done_testing(); diff --git a/src/include/utils/elog.h b/src/include/utils/elog.h index 18a9fed1a6a..7e2a221dbf4 100644 --- a/src/include/utils/elog.h +++ b/src/include/utils/elog.h @@ -502,6 +502,7 @@ extern PGDLLIMPORT int Log_destination; extern PGDLLIMPORT char *Log_destination_string; extern PGDLLIMPORT bool syslog_sequence_numbers; extern PGDLLIMPORT bool syslog_split_messages; +extern PGDLLIMPORT int max_log_size; /* Log destination bitmap */ #define LOG_DESTINATION_STDERR 1 From e41c91a01e47794b45ce9086e078251697c949c5 Mon Sep 17 00:00:00 2001 From: Andrei Liarskii Date: Sat, 2 Nov 2024 08:39:52 +0000 Subject: [PATCH 34/54] Pull request #201: bump llvm to 18 17.0 Merge in MDB/postgres-dev from MDB-31374-bump-llvm-18-pg17_0 to MDB_17_0 Squashed commit of the following: commit 0931fd9d64288653cd950cd19d941bd4d51640d7 Author: Andrey Lyarskiy Date: Tue Oct 29 12:59:04 2024 +0300 bump llvm to 18 --- Dockerfile | 2 +- debian/control | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 76faa3a2045..2fb32956ea4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,7 +30,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ libpam0g-dev \ debhelper debootstrap devscripts make equivs debhelper-compat \ libz-dev flex libicu-dev libio-pty-perl libipc-run-perl libkrb5-dev \ - libldap2-dev liblz4-dev liblz4-tool zstd libperl-dev libreadline-dev libselinux1-dev llvm-dev \ + libldap2-dev liblz4-dev liblz4-tool zstd libperl-dev libreadline-dev libselinux1-dev llvm-18-dev \ libsystemd-dev libxml2-dev libxml2-utils libxslt1-dev \ pkg-config python3-dev systemtap-sdt-dev tcl-dev uuid-dev xsltproc zlib1g-dev \ bison dh-exec docbook-xml docbook-xsl diff --git a/debian/control b/debian/control index dace4c44839..4a98a431946 100644 --- a/debian/control +++ b/debian/control @@ -11,7 +11,7 @@ Rules-Requires-Root: no Build-Depends: autoconf, bison, - clang-15 [!alpha !hppa !hurd-amd64 !hurd-i386 !ia64 !kfreebsd-amd64 !kfreebsd-i386 !loong64 !m68k !powerpc !riscv64 !sh4 !sparc64 !x32] , + clang-18 [!alpha !hppa !hurd-amd64 !hurd-i386 !ia64 !kfreebsd-amd64 !kfreebsd-i386 !loong64 !m68k !powerpc !riscv64 !s390x !sh4 !sparc64 !x32] , debhelper-compat (= 13), dh-exec (>= 0.13~), docbook-xml, @@ -36,7 +36,7 @@ Build-Depends: libxml2-utils, libxslt1-dev, libzstd-dev (>= 1.4.0) , - llvm-15-dev [!alpha !hppa !hurd-amd64 !hurd-i386 !ia64 !kfreebsd-amd64 !kfreebsd-i386 !loong64 !m68k !powerpc !riscv64 !sh4 !sparc64 !x32] , + llvm-18-dev [!alpha !hppa !hurd-amd64 !hurd-i386 !ia64 !kfreebsd-amd64 !kfreebsd-i386 !loong64 !m68k !powerpc !riscv64 !sh4 !sparc64 !x32] , lz4 | liblz4-tool, mawk, perl (>= 5.8), From 017095d95dde87a1aa6d53e847ee95049c5c785e Mon Sep 17 00:00:00 2001 From: diphantxm Date: Mon, 26 Aug 2024 16:17:08 +0300 Subject: [PATCH 35/54] Support FORCE option in analyze command Add new parameter to VACUUM command, FORCE, meaning VACUUM should terminate all backends that prevents the execution by holding conflicting lock --- doc/src/sgml/ref/analyze.sgml | 1 + doc/src/sgml/ref/vacuum.sgml | 11 ++++++ doc/src/sgml/ref/vacuumdb.sgml | 9 +++++ src/backend/commands/async.c | 27 ++++++++++++++ src/backend/commands/vacuum.c | 53 ++++++++++++++++++++++++++-- src/backend/storage/ipc/procsignal.c | 4 +++ src/bin/psql/tab-complete.c | 8 ++--- src/bin/scripts/t/100_vacuumdb.pl | 4 +++ src/bin/scripts/vacuumdb.c | 21 +++++++++++ src/include/commands/async.h | 2 ++ src/include/commands/vacuum.h | 1 + src/include/storage/procsignal.h | 2 ++ src/test/regress/expected/vacuum.out | 6 ++++ src/test/regress/sql/vacuum.sql | 6 ++++ 14 files changed, 149 insertions(+), 6 deletions(-) diff --git a/doc/src/sgml/ref/analyze.sgml b/doc/src/sgml/ref/analyze.sgml index 2b94b378e9f..65d7105829e 100644 --- a/doc/src/sgml/ref/analyze.sgml +++ b/doc/src/sgml/ref/analyze.sgml @@ -28,6 +28,7 @@ ANALYZE [ ( option [, ...] ) ] [ boolean ] SKIP_LOCKED [ boolean ] BUFFER_USAGE_LIMIT size + FORCE [ boolean ] and table_and_columns is: diff --git a/doc/src/sgml/ref/vacuum.sgml b/doc/src/sgml/ref/vacuum.sgml index 9857b35627b..58574d8f0db 100644 --- a/doc/src/sgml/ref/vacuum.sgml +++ b/doc/src/sgml/ref/vacuum.sgml @@ -34,6 +34,7 @@ VACUUM [ ( option [, ...] ) ] [ boolean ] PROCESS_TOAST [ boolean ] + FORCE [ boolean ] TRUNCATE [ boolean ] PARALLEL integer SKIP_DATABASE_STATS [ boolean ] @@ -183,6 +184,16 @@ VACUUM [ ( option [, ...] ) ] [ + + FORCE + + + Specifies that VACUUM should terminate all backends + holding conflicting lock. + + + + INDEX_CLEANUP diff --git a/doc/src/sgml/ref/vacuumdb.sgml b/doc/src/sgml/ref/vacuumdb.sgml index 80c04919a4b..453ee84d07f 100644 --- a/doc/src/sgml/ref/vacuumdb.sgml +++ b/doc/src/sgml/ref/vacuumdb.sgml @@ -368,6 +368,15 @@ PostgreSQL documentation + + + + + Terminate backends holding conflicting lock + + + + diff --git a/src/backend/commands/async.c b/src/backend/commands/async.c index b788ac45e17..f4bfed25794 100644 --- a/src/backend/commands/async.c +++ b/src/backend/commands/async.c @@ -1815,6 +1815,33 @@ HandleNotifyInterrupt(void) SetLatch(MyLatch); } + + +/* + * HandleRvrInterrupt + * + * Signal handler portion of interrupt handling. Let the backend know + * that there's a pending notify interrupt. If we're currently reading + * from the client, this will interrupt the read and + * ProcessClientReadInterrupt() will call ProcessNotifyInterrupt(). + */ +void +HandleRvrInterrupt(void) +{ + /* + * Note: this is called by a SIGNAL HANDLER. You must be very wary what + * you do here. + */ + + /* signal that work needs to be done */ + QueryCancelPending = true; + InterruptPending = true; + ProcDiePending = true; + + /* make sure the event is processed in due course */ + SetLatch(MyLatch); +} + /* * ProcessNotifyInterrupt * diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c index c42c8a680d3..51a34d0cfe5 100644 --- a/src/backend/commands/vacuum.c +++ b/src/backend/commands/vacuum.c @@ -152,6 +152,7 @@ ExecVacuum(ParseState *pstate, VacuumStmt *vacstmt, bool isTopLevel) BufferAccessStrategy bstrategy = NULL; bool verbose = false; bool skip_locked = false; + bool force = false; bool analyze = false; bool freeze = false; bool full = false; @@ -215,6 +216,8 @@ ExecVacuum(ParseState *pstate, VacuumStmt *vacstmt, bool isTopLevel) ring_size = result; } + else if (strcmp(opt->defname, "force") == 0) + force = defGetBoolean(opt); else if (!vacstmt->is_vacuumcmd) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), @@ -307,7 +310,8 @@ ExecVacuum(ParseState *pstate, VacuumStmt *vacstmt, bool isTopLevel) (process_main ? VACOPT_PROCESS_MAIN : 0) | (process_toast ? VACOPT_PROCESS_TOAST : 0) | (skip_database_stats ? VACOPT_SKIP_DATABASE_STATS : 0) | - (only_database_stats ? VACOPT_ONLY_DATABASE_STATS : 0); + (only_database_stats ? VACOPT_ONLY_DATABASE_STATS : 0) | + (force ? VACOPT_FORCE : 0); /* sanity checks on options */ Assert(params.options & (VACOPT_VACUUM | VACOPT_ANALYZE)); @@ -517,6 +521,15 @@ vacuum(List *relations, VacuumParams *params, BufferAccessStrategy bstrategy, errmsg("%s cannot be executed from VACUUM or ANALYZE", stmttype))); + /* sanity check for FORCE */ + if ((params->options & VACOPT_FORCE) != 0) + { + if ((params->options & VACOPT_SKIP_LOCKED) != 0) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("VACUUM option FORCE cannot be used with SKIP_LOCKED"))); + } + /* * Build list of relation(s) to process, putting any new data in * vac_context for safekeeping. @@ -789,6 +802,42 @@ vacuum_open_relation(Oid relid, RangeVar *relation, bits32 options, rel = try_relation_open(relid, lmode); else if (ConditionalLockRelationOid(relid, lmode)) rel = try_relation_open(relid, NoLock); + else if (options & VACOPT_FORCE) + { + LOCKTAG tag; + Oid dbid; + + if (IsSharedRelation(relid)) + dbid = InvalidOid; + else + dbid = MyDatabaseId; + + SET_LOCKTAG_RELATION(tag, dbid, relid); + + while (rel == NULL) + { + VirtualTransactionId* backends = GetLockConflicts(&tag, lmode, NULL); + + /* + * Send signals to all the backends holding the conflicting locks + */ + while (VirtualTransactionIdIsValid(*backends)) + { + SignalVirtualTransaction(*backends, + PROCSIG_CONFLICT_RVR_FORCE, + false); + backends++; + } + rel = try_relation_open(relid, lmode); + if (rel == NULL) + { + ereport(NOTICE, + (errcode(ERRCODE_LOCK_NOT_AVAILABLE), + errmsg("retrying attemts of acquiring lock for \"%s\" --- lock not available", + relation->relname))); + } + } + } else { rel = NULL; @@ -912,7 +961,7 @@ expand_vacuum_rel(VacuumRelation *vrel, MemoryContext vac_context, * below, as well as find_all_inheritors's expectation that the caller * holds some lock on the starting relation. */ - rvr_opts = (options & VACOPT_SKIP_LOCKED) ? RVR_SKIP_LOCKED : 0; + rvr_opts = ((options & VACOPT_SKIP_LOCKED) ? RVR_SKIP_LOCKED : 0); relid = RangeVarGetRelidExtended(vrel->relation, AccessShareLock, rvr_opts, diff --git a/src/backend/storage/ipc/procsignal.c b/src/backend/storage/ipc/procsignal.c index d6857f5a8bb..c8230d4a30a 100644 --- a/src/backend/storage/ipc/procsignal.c +++ b/src/backend/storage/ipc/procsignal.c @@ -680,5 +680,9 @@ procsignal_sigusr1_handler(SIGNAL_ARGS) if (CheckProcSignal(PROCSIG_RECOVERY_CONFLICT_BUFFERPIN)) HandleRecoveryConflictInterrupt(PROCSIG_RECOVERY_CONFLICT_BUFFERPIN); + /* MDB additions */ + if (CheckProcSignal(PROCSIG_CONFLICT_RVR_FORCE)) + HandleRvrInterrupt(); + SetLatch(MyLatch); } diff --git a/src/bin/psql/tab-complete.c b/src/bin/psql/tab-complete.c index d4c5f6c3798..1d970d5d609 100644 --- a/src/bin/psql/tab-complete.c +++ b/src/bin/psql/tab-complete.c @@ -2721,8 +2721,8 @@ psql_completion(const char *text, int start, int end) * one word, so the above test is correct. */ if (ends_with(prev_wd, '(') || ends_with(prev_wd, ',')) - COMPLETE_WITH("VERBOSE", "SKIP_LOCKED", "BUFFER_USAGE_LIMIT"); - else if (TailMatches("VERBOSE|SKIP_LOCKED")) + COMPLETE_WITH("VERBOSE", "SKIP_LOCKED", "BUFFER_USAGE_LIMIT", "FORCE"); + else if (TailMatches("VERBOSE|SKIP_LOCKED|FORCE")) COMPLETE_WITH("ON", "OFF"); } else if (HeadMatches("ANALYZE") && TailMatches("(")) @@ -4764,8 +4764,8 @@ psql_completion(const char *text, int start, int end) "DISABLE_PAGE_SKIPPING", "SKIP_LOCKED", "INDEX_CLEANUP", "PROCESS_MAIN", "PROCESS_TOAST", "TRUNCATE", "PARALLEL", "SKIP_DATABASE_STATS", - "ONLY_DATABASE_STATS", "BUFFER_USAGE_LIMIT"); - else if (TailMatches("FULL|FREEZE|ANALYZE|VERBOSE|DISABLE_PAGE_SKIPPING|SKIP_LOCKED|PROCESS_MAIN|PROCESS_TOAST|TRUNCATE|SKIP_DATABASE_STATS|ONLY_DATABASE_STATS")) + "ONLY_DATABASE_STATS", "BUFFER_USAGE_LIMIT", "FORCE"); + else if (TailMatches("FULL|FREEZE|ANALYZE|VERBOSE|DISABLE_PAGE_SKIPPING|SKIP_LOCKED|PROCESS_MAIN|PROCESS_TOAST|TRUNCATE|SKIP_DATABASE_STATS|ONLY_DATABASE_STATS|FORCE")) COMPLETE_WITH("ON", "OFF"); else if (TailMatches("INDEX_CLEANUP")) COMPLETE_WITH("AUTO", "ON", "OFF"); diff --git a/src/bin/scripts/t/100_vacuumdb.pl b/src/bin/scripts/t/100_vacuumdb.pl index 1a2bcb49591..5ccd5cf069f 100644 --- a/src/bin/scripts/t/100_vacuumdb.pl +++ b/src/bin/scripts/t/100_vacuumdb.pl @@ -48,6 +48,10 @@ [ 'vacuumdb', '--skip-locked', '--analyze-only', 'postgres' ], qr/statement: ANALYZE \(SKIP_LOCKED\).*;/, 'vacuumdb --skip-locked --analyze-only'); +$node->issues_sql_like( + [ 'vacuumdb', '--force', '--analyze-only', 'postgres' ], + qr/statement: ANALYZE \(FORCE\).*;/, + 'vacuumdb --force --analyze-only'); $node->command_fails( [ 'vacuumdb', '--analyze-only', '--disable-page-skipping', 'postgres' ], '--analyze-only and --disable-page-skipping specified together'); diff --git a/src/bin/scripts/vacuumdb.c b/src/bin/scripts/vacuumdb.c index 5aff0dd25d5..2ac12eb11e3 100644 --- a/src/bin/scripts/vacuumdb.c +++ b/src/bin/scripts/vacuumdb.c @@ -47,6 +47,7 @@ typedef struct vacuumingOptions bool process_toast; bool skip_database_stats; char *buffer_usage_limit; + bool force; } vacuumingOptions; /* object filter options */ @@ -128,6 +129,7 @@ main(int argc, char *argv[]) {"no-process-toast", no_argument, NULL, 11}, {"no-process-main", no_argument, NULL, 12}, {"buffer-usage-limit", required_argument, NULL, 13}, + {"force", no_argument, NULL, 14}, {NULL, 0, NULL, 0} }; @@ -275,6 +277,9 @@ main(int argc, char *argv[]) case 13: vacopts.buffer_usage_limit = escape_quotes(optarg); break; + case 14: + vacopts.force = true; + break; default: /* getopt_long already emitted a complaint */ pg_log_error_hint("Try \"%s --help\" for more information.", progname); @@ -360,6 +365,7 @@ main(int argc, char *argv[]) pg_fatal("cannot use the \"%s\" option with the \"%s\" option", "buffer-usage-limit", "full"); + /* fill cparams except for dbname, which is set below */ cparams.pghost = host; cparams.pgport = port; @@ -996,6 +1002,13 @@ prepare_vacuum_command(PQExpBuffer sql, int serverVersion, appendPQExpBuffer(sql, "%sSKIP_LOCKED", sep); sep = comma; } + if (vacopts->force) + { + ///* FORCE is supported since v12 */ + Assert(serverVersion >= 120000); + appendPQExpBuffer(sql, "%sFORCE", sep); + sep = comma; + } if (vacopts->verbose) { appendPQExpBuffer(sql, "%sVERBOSE", sep); @@ -1082,6 +1095,13 @@ prepare_vacuum_command(PQExpBuffer sql, int serverVersion, appendPQExpBuffer(sql, "%sSKIP_LOCKED", sep); sep = comma; } + if (vacopts->force) + { + ///* FORCE is supported since v12 */ + Assert(serverVersion >= 120000); + appendPQExpBuffer(sql, "%sFORCE", sep); + sep = comma; + } if (vacopts->full) { appendPQExpBuffer(sql, "%sFULL", sep); @@ -1191,6 +1211,7 @@ help(const char *progname) printf(_(" -P, --parallel=PARALLEL_WORKERS use this many background workers for vacuum, if available\n")); printf(_(" -q, --quiet don't write any messages\n")); printf(_(" --skip-locked skip relations that cannot be immediately locked\n")); + printf(_(" --force terminate backends holding conflicting lock\n")); printf(_(" -t, --table='TABLE[(COLUMNS)]' vacuum specific table(s) only\n")); printf(_(" -v, --verbose write a lot of output\n")); printf(_(" -V, --version output version information, then exit\n")); diff --git a/src/include/commands/async.h b/src/include/commands/async.h index 0f2082b1304..2321dca6ba1 100644 --- a/src/include/commands/async.h +++ b/src/include/commands/async.h @@ -43,6 +43,8 @@ extern void AtPrepare_Notify(void); /* signal handler for inbound notifies (PROCSIG_NOTIFY_INTERRUPT) */ extern void HandleNotifyInterrupt(void); +extern void HandleRvrInterrupt(void); + /* process interrupts */ extern void ProcessNotifyInterrupt(bool flush); diff --git a/src/include/commands/vacuum.h b/src/include/commands/vacuum.h index 759f9a87d38..b75f4c69867 100644 --- a/src/include/commands/vacuum.h +++ b/src/include/commands/vacuum.h @@ -188,6 +188,7 @@ typedef struct VacAttrStats #define VACOPT_DISABLE_PAGE_SKIPPING 0x100 /* don't skip any pages */ #define VACOPT_SKIP_DATABASE_STATS 0x200 /* skip vac_update_datfrozenxid() */ #define VACOPT_ONLY_DATABASE_STATS 0x400 /* only vac_update_datfrozenxid() */ +#define VACOPT_FORCE 0x800 /* terminate conflicting backend if cannot get lock */ /* * Values used by index_cleanup and truncate params. diff --git a/src/include/storage/procsignal.h b/src/include/storage/procsignal.h index 58a042f1b9f..3003937cf0a 100644 --- a/src/include/storage/procsignal.h +++ b/src/include/storage/procsignal.h @@ -49,6 +49,8 @@ typedef enum PROCSIG_RECOVERY_CONFLICT_LAST = PROCSIG_RECOVERY_CONFLICT_STARTUP_DEADLOCK, PROCSIG_SLOTSYNC_MESSAGE, /* ask slot synchronization to stop */ + /* MDB additions */ + PROCSIG_CONFLICT_RVR_FORCE, NUM_PROCSIGNALS /* Must be last! */ } ProcSignalReason; diff --git a/src/test/regress/expected/vacuum.out b/src/test/regress/expected/vacuum.out index 7518806e61f..768b8377261 100644 --- a/src/test/regress/expected/vacuum.out +++ b/src/test/regress/expected/vacuum.out @@ -327,6 +327,12 @@ VACUUM (SKIP_LOCKED) vactst; VACUUM (SKIP_LOCKED, FULL) vactst; ANALYZE (SKIP_LOCKED) vactst; RESET client_min_messages; +-- FORCE option +ANALYZE (FORCE) vactst; +VACUUM (ANALYZE, FORCE) vactst; +VACUUM (FORCE) vactst; +VACUUM (ANALYZE, FORCE, SKIP_LOCKED) vactst; +ERROR: VACUUM option FORCE cannot be used with SKIP_LOCKED -- ensure VACUUM and ANALYZE don't have a problem with serializable SET default_transaction_isolation = serializable; VACUUM vactst; diff --git a/src/test/regress/sql/vacuum.sql b/src/test/regress/sql/vacuum.sql index d272dd064eb..7da2a8227e4 100644 --- a/src/test/regress/sql/vacuum.sql +++ b/src/test/regress/sql/vacuum.sql @@ -264,6 +264,12 @@ VACUUM (SKIP_LOCKED, FULL) vactst; ANALYZE (SKIP_LOCKED) vactst; RESET client_min_messages; +-- FORCE option +ANALYZE (FORCE) vactst; +VACUUM (ANALYZE, FORCE) vactst; +VACUUM (FORCE) vactst; +VACUUM (ANALYZE, FORCE, SKIP_LOCKED) vactst; + -- ensure VACUUM and ANALYZE don't have a problem with serializable SET default_transaction_isolation = serializable; VACUUM vactst; From 205071dd6a303fcae3bb278c1b47af9f36cb9d5d Mon Sep 17 00:00:00 2001 From: diphantxm Date: Tue, 12 Nov 2024 15:45:28 +0300 Subject: [PATCH 36/54] truncate query to be logged in simple query --- src/backend/tcop/postgres.c | 10 ++++++++-- src/backend/utils/error/elog.c | 36 ++++++++++++++++++++++++++++------ src/include/utils/elog.h | 1 + 3 files changed, 39 insertions(+), 8 deletions(-) diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c index 00d4073c089..20ccfb70741 100644 --- a/src/backend/tcop/postgres.c +++ b/src/backend/tcop/postgres.c @@ -72,6 +72,7 @@ #include "tcop/pquery.h" #include "tcop/tcopprot.h" #include "tcop/utility.h" +#include "utils/elog.h" #include "utils/guc_hooks.h" #include "utils/injection_point.h" #include "utils/lsyscache.h" @@ -1025,11 +1026,14 @@ exec_simple_query(const char *query_string) bool was_logged = false; bool use_implicit_block; char msec_str[32]; + char* query_log; /* * Report query to various monitoring facilities. */ debug_query_string = query_string; + bool copied = false; + query_log = build_query_log(query_string, &copied); pgstat_report_activity(STATE_RUNNING, query_string); @@ -1074,7 +1078,7 @@ exec_simple_query(const char *query_string) if (check_log_statement(parsetree_list)) { ereport(LOG, - (errmsg("statement: %s", query_string), + (errmsg("statement: %s", query_log), errhidestmt(true), errdetail_execute(parsetree_list))); was_logged = true; @@ -1373,7 +1377,7 @@ exec_simple_query(const char *query_string) case 2: ereport(LOG, (errmsg("duration: %s ms statement: %s", - msec_str, query_string), + msec_str, query_log), errhidestmt(true), errdetail_execute(parsetree_list))); break; @@ -1384,6 +1388,8 @@ exec_simple_query(const char *query_string) TRACE_POSTGRESQL_QUERY_DONE(query_string); + if (query_log && copied) + pfree(query_log); debug_query_string = NULL; } diff --git a/src/backend/utils/error/elog.c b/src/backend/utils/error/elog.c index 77bbd9230dc..741540b2457 100644 --- a/src/backend/utils/error/elog.c +++ b/src/backend/utils/error/elog.c @@ -1694,12 +1694,9 @@ EmitErrorReport(void) CHECK_STACK_DEPTH(); oldcontext = MemoryContextSwitchTo(edata->assoc_context); - if (max_log_size != 0 && debug_query_string != NULL) - { - char* str = debug_query_string; - str[pg_mbcliplen(str, strlen(str), max_log_size)] = '\0'; - debug_query_string = str; - } + const char* old_query_string = debug_query_string; + bool copied = false; + debug_query_string = build_query_log(debug_query_string, &copied); /* * Reset the formatted timestamp fields before emitting any logs. This @@ -1741,6 +1738,12 @@ EmitErrorReport(void) MemoryContextSwitchTo(oldcontext); recursion_depth--; + + if (debug_query_string && copied) + { + pfree(debug_query_string); + debug_query_string = old_query_string; + } } /* @@ -3788,3 +3791,24 @@ vwrite_stderr(const char *fmt, va_list ap) } #endif } + +char* +build_query_log(const char* query, bool *copied) +{ + *copied = false; + if (!query) + return NULL; + + size_t query_len = strlen(query); + if (max_log_size == 0 || query_len < max_log_size) + { + return query; + } + + *copied = true; + size_t query_log_len = pg_mbcliplen(query, query_len, max_log_size); + char* query_log = (char*)palloc(query_log_len+1); + memcpy(query_log, query, query_log_len); + query_log[query_log_len] = '\0'; + return query_log; +} diff --git a/src/include/utils/elog.h b/src/include/utils/elog.h index 7e2a221dbf4..56d20004086 100644 --- a/src/include/utils/elog.h +++ b/src/include/utils/elog.h @@ -517,6 +517,7 @@ extern void log_status_format(StringInfo buf, const char *format, extern void DebugFileOpen(void); extern char *unpack_sql_state(int sql_state); extern bool in_error_recursion_trouble(void); +char* build_query_log(const char* query, bool *copied); /* Common functions shared across destinations */ extern void reset_formatted_start_time(void); From 5c36802d18351751af32297113abc0f192dccceb Mon Sep 17 00:00:00 2001 From: Andrey Borodin Date: Wed, 13 Nov 2024 12:03:52 +0500 Subject: [PATCH 37/54] GUCify NUM_BUFFER_PARTITIONS Also truncate query to be logged in simple query Remove unused functions in buf_internals.h --- src/backend/storage/buffer/bufmgr.c | 8 ++++++++ src/backend/utils/init/globals.c | 4 ++++ src/backend/utils/misc/guc_tables.c | 11 +++++++++++ src/backend/utils/misc/postgresql.conf.sample | 1 + src/include/storage/buf_internals.h | 14 +------------- src/include/storage/lwlock.h | 4 +++- src/include/utils/guc_hooks.h | 1 + src/test/modules/test_misc/t/003_check_guc.pl | 1 + 8 files changed, 30 insertions(+), 14 deletions(-) diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c index 7daf1bed041..37b7a8ce7d6 100644 --- a/src/backend/storage/buffer/bufmgr.c +++ b/src/backend/storage/buffer/bufmgr.c @@ -59,6 +59,7 @@ #include "storage/proc.h" #include "storage/smgr.h" #include "storage/standby.h" +#include "utils/guc_hooks.h" #include "utils/memdebug.h" #include "utils/ps_status.h" #include "utils/rel.h" @@ -144,6 +145,13 @@ int bgwriter_lru_maxpages = 100; double bgwriter_lru_multiplier = 2.0; bool track_io_timing = false; +/* GUC assign hook for num_buffer_partitions_log2 */ +void +assign_num_buffer_partitions_log2(int newval, void *extra) +{ + num_buffer_partitions_mask = (1 << newval) - 1; +} + /* * How many buffers PrefetchBuffer callers should try to stay ahead of their * ReadBuffer calls by. Zero means "never prefetch". This value is only used diff --git a/src/backend/utils/init/globals.c b/src/backend/utils/init/globals.c index cc61937eef7..5be5684c0d1 100644 --- a/src/backend/utils/init/globals.c +++ b/src/backend/utils/init/globals.c @@ -166,3 +166,7 @@ int notify_buffers = 16; int serializable_buffers = 32; int subtransaction_buffers = 0; int transaction_buffers = 0; + +/* shared buffers partitions number and mask */ +int num_buffer_partitions_log2 = 7; +int num_buffer_partitions_mask = 127; \ No newline at end of file diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index 63d28cae8e9..c25ff52a585 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -2523,6 +2523,17 @@ struct config_int ConfigureNamesInt[] = check_max_stack_depth, assign_max_stack_depth, NULL }, + { + /* MDB prefix here is needed in case when vanilla is starting with config with this value */ + {"ycmdb.num_buffer_partitions_log2", PGC_POSTMASTER, RESOURCES_MEM, + gettext_noop("Sets number of partitions for shared buffers mapping hashtable."), + NULL, + }, + &num_buffer_partitions_log2, + 7, 5, 16, + NULL, assign_num_buffer_partitions_log2, NULL + }, + { {"temp_file_limit", PGC_SUSET, RESOURCES_DISK, gettext_noop("Limits the total size of all temporary files used by each process."), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index 95c086d2b4b..022987b5557 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -143,6 +143,7 @@ #autovacuum_work_mem = -1 # min 64kB, or -1 to use maintenance_work_mem #logical_decoding_work_mem = 64MB # min 64kB #max_stack_depth = 2MB # min 100kB +#mdb.num_buffer_partitions_log2 = 7 # number of partitions in shared buffer mapping hashtable #shared_memory_type = mmap # the default is the first option # supported by the operating system: # mmap diff --git a/src/include/storage/buf_internals.h b/src/include/storage/buf_internals.h index f190e6e5e46..14421647c23 100644 --- a/src/include/storage/buf_internals.h +++ b/src/include/storage/buf_internals.h @@ -176,23 +176,11 @@ BufTagMatchesRelFileLocator(const BufferTag *tag, * hash code with BufTableHashCode(), then apply BufMappingPartitionLock(). * NB: NUM_BUFFER_PARTITIONS must be a power of 2! */ -static inline uint32 -BufTableHashPartition(uint32 hashcode) -{ - return hashcode % NUM_BUFFER_PARTITIONS; -} - static inline LWLock * BufMappingPartitionLock(uint32 hashcode) { return &MainLWLockArray[BUFFER_MAPPING_LWLOCK_OFFSET + - BufTableHashPartition(hashcode)].lock; -} - -static inline LWLock * -BufMappingPartitionLockByIndex(uint32 index) -{ - return &MainLWLockArray[BUFFER_MAPPING_LWLOCK_OFFSET + index].lock; + (hashcode & num_buffer_partitions_mask)].lock; } /* diff --git a/src/include/storage/lwlock.h b/src/include/storage/lwlock.h index 5435948a1cd..dfa51ddbb77 100644 --- a/src/include/storage/lwlock.h +++ b/src/include/storage/lwlock.h @@ -90,7 +90,9 @@ extern PGDLLIMPORT int NamedLWLockTrancheRequests; */ /* Number of partitions of the shared buffer mapping hashtable */ -#define NUM_BUFFER_PARTITIONS 128 +extern int num_buffer_partitions_log2; +extern int num_buffer_partitions_mask; +#define NUM_BUFFER_PARTITIONS (1 << num_buffer_partitions_log2) /* Number of partitions the shared lock tables are divided into */ #define LOG2_NUM_LOCK_PARTITIONS 4 diff --git a/src/include/utils/guc_hooks.h b/src/include/utils/guc_hooks.h index 1babff78bf3..656d6e69860 100644 --- a/src/include/utils/guc_hooks.h +++ b/src/include/utils/guc_hooks.h @@ -91,6 +91,7 @@ extern bool check_max_worker_processes(int *newval, void **extra, GucSource source); extern bool check_max_stack_depth(int *newval, void **extra, GucSource source); extern void assign_max_stack_depth(int newval, void *extra); +extern void assign_num_buffer_partitions_log2(int newval, void *extra); extern bool check_multixact_member_buffers(int *newval, void **extra, GucSource source); extern bool check_multixact_offset_buffers(int *newval, void **extra, diff --git a/src/test/modules/test_misc/t/003_check_guc.pl b/src/test/modules/test_misc/t/003_check_guc.pl index e754778c1e8..67e09d11ccc 100644 --- a/src/test/modules/test_misc/t/003_check_guc.pl +++ b/src/test/modules/test_misc/t/003_check_guc.pl @@ -73,6 +73,7 @@ } push @gucs_in_file, "ycmdb.yc_grant_checker"; +push @gucs_in_file, "ycmdb.num_buffer_partitions_log2"; close $contents; From fc84bbe0b2283cc76772ebac4ede7f6979b8a0ca Mon Sep 17 00:00:00 2001 From: Kirill Reshke Date: Wed, 11 Dec 2024 20:14:55 +0300 Subject: [PATCH 38/54] Change storage class for mdb_replication utilities --- src/include/replication/slot.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/include/replication/slot.h b/src/include/replication/slot.h index 21159782f36..8b3a56916b0 100644 --- a/src/include/replication/slot.h +++ b/src/include/replication/slot.h @@ -305,11 +305,11 @@ extern void CheckRoleMDBReplSlotPermissions(bool role_has_rolreplication, bool i */ extern void CheckRoleUseMDBReservedName(const char* name, bool role_has_rolreplication); -inline void CheckMDBReservedName(const char* name) { +static inline void CheckMDBReservedName(const char* name) { CheckRoleUseMDBReservedName(name, has_rolreplication(GetUserId())); } -inline void CheckMDBReplSlotPermissions(void) { +static inline void CheckMDBReplSlotPermissions(void) { Oid role; role = get_role_oid("mdb_replication", /* missing ok*/ true); return CheckRoleMDBReplSlotPermissions(has_rolreplication(GetUserId()), is_member_of_role(GetUserId(), role)); From 4f17999e035ba19965e9b326983168c3554bac5e Mon Sep 17 00:00:00 2001 From: reshke Date: Thu, 16 Jan 2025 19:19:02 +0300 Subject: [PATCH 39/54] Add CI Do not run mdb locales tests in OS Fix CI && fast CI circuit (#24) Use mirror apt repo (#35) Refactor regress Dockerfile Update Dockerfile: try Update Dockerfile --- .github/workflows/regress.yml | 29 ++++++++++++++++++++++ Dockerfile | 5 +++- docker/regress/Dockerfile | 46 +++++++++++++++++++++++++++++++++++ docker/regress/run_tests.sh | 6 +++++ docker/regress/run_tests_f.sh | 6 +++++ 5 files changed, 91 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/regress.yml create mode 100644 docker/regress/Dockerfile create mode 100755 docker/regress/run_tests.sh create mode 100755 docker/regress/run_tests_f.sh diff --git a/.github/workflows/regress.yml b/.github/workflows/regress.yml new file mode 100644 index 00000000000..5a565f4160c --- /dev/null +++ b/.github/workflows/regress.yml @@ -0,0 +1,29 @@ +name: Docker Image CI + +on: + push: + branches: [ "MDB_*" ] + pull_request: + branches: [ "MDB_*" ] + +jobs: + + check: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Build the Docker image + run: docker build . --file docker/regress/Dockerfile --tag regress_test:1234 && docker run --entrypoint /home/build-user/docker/regress/run_tests_f.sh regress_test:1234 + + check-world: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Build the Docker image + run: docker build . --file docker/regress/Dockerfile --tag regress_test:1234 && docker run regress_test:1234 + + diff --git a/Dockerfile b/Dockerfile index 2fb32956ea4..3560130b8eb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,6 +23,8 @@ RUN echo "deb http://dist.yandex.ru/mdb-${CODE_NAME}-secure stable/\$(ARCH)/" >> RUN curl -s 'http://keyserver.ubuntu.com/pks/lookup?op=get&search=0xafc3ce0d00e3c45a357e9e637fcd11186050cd1a' | \ gpg --dearmour -o /etc/apt/trusted.gpg.d/yandex.gpg +RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys FF5F4D0E27393420 + RUN apt-get update && apt-get install -y --no-install-recommends \ sudo build-essential \ gcc lsb-release libssl-dev gnupg openssl \ @@ -38,7 +40,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ RUN apt-get install -y \ libmdblocales1 libmdblocales-dev \ postgresql-client-common=${PGDG_VER} \ - postgresql-common=${PGDG_VER} + postgresql-common=${PGDG_VER} \ + postgresql-common-dev=${PGDG_VER} RUN groupadd -g 999 build-user && \ useradd -r -u 999 -g build-user build-user diff --git a/docker/regress/Dockerfile b/docker/regress/Dockerfile new file mode 100644 index 00000000000..f616102fe7e --- /dev/null +++ b/docker/regress/Dockerfile @@ -0,0 +1,46 @@ +ARG codename +FROM ubuntu:${codename:-bionic} + +ARG codename +ENV CODE_NAME=${codename:-bionic} + +ARG pgdg +ENV PGDG_VER=${pgdg:-242-2-pgdg18.04+1+yandex220} + +ENV DEBIAN_FRONTEND=noninteractive +ENV TZ=Europe/Moskow +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + +RUN apt update && apt install -y ca-certificates +RUN sed -i 's/archive.ubuntu.com/mirror.yandex.ru/g' /etc/apt/sources.list +RUN apt-get update && apt-get install -y --no-install-recommends \ + sudo build-essential \ + gcc lsb-release libssl-dev gnupg openssl \ + gdb git curl ca-certificates + +RUN curl -s 'http://keyserver.ubuntu.com/pks/lookup?op=get&search=0xafc3ce0d00e3c45a357e9e637fcd11186050cd1a' | \ + gpg --dearmour -o /etc/apt/trusted.gpg.d/yandex.gpg + +RUN apt-get update && apt-get install -y --no-install-recommends \ + sudo build-essential \ + gcc lsb-release libssl-dev gnupg openssl \ + gdb git \ + libpam0g-dev \ + debhelper debootstrap devscripts make equivs debhelper-compat \ + libz-dev flex libicu-dev libio-pty-perl libipc-run-perl libkrb5-dev \ + libldap2-dev liblz4-dev liblz4-tool zstd libperl-dev libreadline-dev libselinux1-dev llvm-dev \ + libsystemd-dev libxml2-dev libxml2-utils libxslt1-dev \ + pkg-config python3-dev systemtap-sdt-dev tcl-dev uuid-dev xsltproc zlib1g-dev \ + bison dh-exec docbook-xml docbook-xsl + +RUN groupadd -g 999 build-user && \ + useradd -r -u 999 -g build-user build-user + +COPY . /home/build-user +RUN chown build-user:build-user /home -R && usermod -aG sudo build-user + +RUN echo 'build-user ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers + +USER build-user + +ENTRYPOINT ["/home/build-user/docker/regress/run_tests.sh"] diff --git a/docker/regress/run_tests.sh b/docker/regress/run_tests.sh new file mode 100755 index 00000000000..5b72eaea60d --- /dev/null +++ b/docker/regress/run_tests.sh @@ -0,0 +1,6 @@ +#!/bin/bash +set -ex + +sed -i '/mdb-related/,$d' src/test/regress/*/misc.* + +make check-world diff --git a/docker/regress/run_tests_f.sh b/docker/regress/run_tests_f.sh new file mode 100755 index 00000000000..a7ff7f136aa --- /dev/null +++ b/docker/regress/run_tests_f.sh @@ -0,0 +1,6 @@ +#!/bin/bash +set -ex + +sed -i '/mdb-related/,$d' src/test/regress/*/misc.* + +make check From 3a5c5258136dc5740655d7c943b4e0b72ccc5f65 Mon Sep 17 00:00:00 2001 From: reshke Date: Tue, 21 Jan 2025 16:14:43 +0500 Subject: [PATCH 40/54] Do not use schema public in mdb_superuser regression tests (#7) --- src/test/regress/expected/mdb_superuser.out | 5 +++-- src/test/regress/sql/mdb_superuser.sql | 6 ++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/test/regress/expected/mdb_superuser.out b/src/test/regress/expected/mdb_superuser.out index 8d989661434..7849311cd1d 100644 --- a/src/test/regress/expected/mdb_superuser.out +++ b/src/test/regress/expected/mdb_superuser.out @@ -85,6 +85,7 @@ CREATE DATABASE regress_check_owner OWNER regress_mdb_superuser_user2; SET ROLE regress_mdb_superuser_user2; CREATE SCHEMA regtest; CREATE TABLE regtest.regtest(); +GRANT CREATE ON SCHEMA regtest TO regress_mdb_su_role_o2; -- this should fail SET ROLE regress_mdb_superuser_user3; GRANT ALL ON TABLE regtest.regtest TO regress_mdb_superuser_user3; @@ -96,9 +97,9 @@ GRANT ALL ON TABLE regtest.regtest TO regress_mdb_superuser_user1; ALTER TABLE regtest.regtest OWNER TO regress_mdb_superuser_user1; -- Check grantor SET ROLE regress_mdb_su_role_o2; -CREATE TABLE public.role_o2_t(); +CREATE TABLE regtest.role_o2_t(); SET ROLE mdb_superuser; -GRANT SELECT ON public.role_o2_t TO regress_mdb_su_role_o1; +GRANT SELECT ON regtest.role_o2_t TO regress_mdb_su_role_o1; SELECT grantor from information_schema.role_table_grants diff --git a/src/test/regress/sql/mdb_superuser.sql b/src/test/regress/sql/mdb_superuser.sql index b0e702c630c..0877d9258f6 100644 --- a/src/test/regress/sql/mdb_superuser.sql +++ b/src/test/regress/sql/mdb_superuser.sql @@ -113,6 +113,8 @@ SET ROLE regress_mdb_superuser_user2; CREATE SCHEMA regtest; CREATE TABLE regtest.regtest(); +GRANT CREATE ON SCHEMA regtest TO regress_mdb_su_role_o2; + -- this should fail SET ROLE regress_mdb_superuser_user3; @@ -127,11 +129,11 @@ ALTER TABLE regtest.regtest OWNER TO regress_mdb_superuser_user1; SET ROLE regress_mdb_su_role_o2; -CREATE TABLE public.role_o2_t(); +CREATE TABLE regtest.role_o2_t(); SET ROLE mdb_superuser; -GRANT SELECT ON public.role_o2_t TO regress_mdb_su_role_o1; +GRANT SELECT ON regtest.role_o2_t TO regress_mdb_su_role_o1; SELECT grantor From e2ba37541cd779f54ba1eb1afeca6d45a4de52fb Mon Sep 17 00:00:00 2001 From: Yury Frolov <57130330+EinKrebs@users.noreply.github.com> Date: Fri, 24 Jan 2025 16:05:48 +0500 Subject: [PATCH 41/54] Rewrite Dockerfile logic --- .github/workflows/regress.yml | 2 +- docker/regress/Dockerfile | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/regress.yml b/.github/workflows/regress.yml index 5a565f4160c..c8b48bcfb3c 100644 --- a/.github/workflows/regress.yml +++ b/.github/workflows/regress.yml @@ -24,6 +24,6 @@ jobs: steps: - uses: actions/checkout@v4 - name: Build the Docker image - run: docker build . --file docker/regress/Dockerfile --tag regress_test:1234 && docker run regress_test:1234 + run: docker build . --file docker/regress/Dockerfile --tag regress_test:1234 && docker run --entrypoint /home/build-user/docker/regress/run_tests.sh regress_test:1234 diff --git a/docker/regress/Dockerfile b/docker/regress/Dockerfile index f616102fe7e..67ee2d63644 100644 --- a/docker/regress/Dockerfile +++ b/docker/regress/Dockerfile @@ -4,9 +4,6 @@ FROM ubuntu:${codename:-bionic} ARG codename ENV CODE_NAME=${codename:-bionic} -ARG pgdg -ENV PGDG_VER=${pgdg:-242-2-pgdg18.04+1+yandex220} - ENV DEBIAN_FRONTEND=noninteractive ENV TZ=Europe/Moskow RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone @@ -42,5 +39,8 @@ RUN chown build-user:build-user /home -R && usermod -aG sudo build-user RUN echo 'build-user ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers USER build-user +WORKDIR /home/build-user -ENTRYPOINT ["/home/build-user/docker/regress/run_tests.sh"] +RUN CFLAGS="" ./configure --prefix=/home/build-user/pgbin --without-mdblocales --enable-depend --enable-cassert --enable-debug --enable-tap-tests && \ + make -j8 && \ + sudo make install From 18d384877e6e3c1296b63630dd1a69f649396495 Mon Sep 17 00:00:00 2001 From: reshke Date: Fri, 7 Feb 2025 13:07:37 +0000 Subject: [PATCH 42/54] Introduce mdb_read_all_data/mdb_write_all_data Allow usage on schema for mdb_read_all_data (#23) --- src/backend/catalog/aclchk.c | 39 +++++++ .../regress/expected/mdb_read_write_roles.out | 97 +++++++++++++++++ src/test/regress/expected/mdb_superuser.out | 4 +- src/test/regress/expected/test_setup.out | 3 + src/test/regress/parallel_schedule | 2 + src/test/regress/sql/mdb_read_write_roles.sql | 101 ++++++++++++++++++ src/test/regress/sql/mdb_superuser.sql | 5 +- src/test/regress/sql/test_setup.sql | 3 + 8 files changed, 249 insertions(+), 5 deletions(-) create mode 100644 src/test/regress/expected/mdb_read_write_roles.out create mode 100644 src/test/regress/sql/mdb_read_write_roles.sql diff --git a/src/backend/catalog/aclchk.c b/src/backend/catalog/aclchk.c index bc0e259f69f..eb1c2cf663d 100644 --- a/src/backend/catalog/aclchk.c +++ b/src/backend/catalog/aclchk.c @@ -3346,6 +3346,8 @@ pg_class_aclmask_ext(Oid table_oid, Oid roleid, AclMode mask, bool isNull; Acl *acl; Oid ownerId; + Oid mdb_read_all_data_oid; + Oid mdb_write_all_data_oid; /* * Must get the relation's tuple from pg_class @@ -3396,6 +3398,9 @@ pg_class_aclmask_ext(Oid table_oid, Oid roleid, AclMode mask, */ ownerId = classForm->relowner; + mdb_read_all_data_oid = get_role_oid("mdb_read_all_data", true); + mdb_write_all_data_oid = get_role_oid("mdb_write_all_data", true); + aclDatum = SysCacheGetAttr(RELOID, tuple, Anum_pg_class_relacl, &isNull); if (isNull) @@ -3435,6 +3440,17 @@ pg_class_aclmask_ext(Oid table_oid, Oid roleid, AclMode mask, has_privs_of_role(roleid, ROLE_PG_READ_ALL_DATA)) result |= ACL_SELECT; + + /* + * Check if ACL_SELECT is being checked and, if so, and not set already as + * part of the result, then check if the user is a member of the + * mdb_read_all_data role, and this is not some dangerous relation to grant SELECT to + */ + if (mask & ACL_SELECT && !(result & ACL_SELECT) && + has_privs_of_role(roleid, mdb_read_all_data_oid) && + !has_privs_of_unwanted_system_role(ownerId)) + result |= ACL_SELECT; + /* * Check if ACL_INSERT, ACL_UPDATE, or ACL_DELETE is being checked and, if * so, and not set already as part of the result, then check if the user @@ -3447,6 +3463,19 @@ pg_class_aclmask_ext(Oid table_oid, Oid roleid, AclMode mask, has_privs_of_role(roleid, ROLE_PG_WRITE_ALL_DATA)) result |= (mask & (ACL_INSERT | ACL_UPDATE | ACL_DELETE)); + + /* + * Check if ACL_INSERT, ACL_UPDATE, or ACL_DELETE is being checked and, if + * so, and not set already as part of the result, then check if the user + * is a member of the mdb_write_all_data role, and this is not some + * dangerous relation to grant write access. + */ + if (mask & (ACL_INSERT | ACL_UPDATE | ACL_DELETE) && + !(result & (ACL_INSERT | ACL_UPDATE | ACL_DELETE)) && + has_privs_of_role(roleid, mdb_write_all_data_oid) && + !has_privs_of_unwanted_system_role(ownerId)) + result |= (mask & (ACL_INSERT | ACL_UPDATE | ACL_DELETE)); + /* * Check if ACL_MAINTAIN is being checked and, if so, and not already set * as part of the result, then check if the user is a member of the @@ -3672,6 +3701,7 @@ pg_namespace_aclmask_ext(Oid nsp_oid, Oid roleid, bool isNull; Acl *acl; Oid ownerId; + Oid mdb_read_all_data_oid; /* Superusers bypass all permission checking. */ if (superuser_arg(roleid)) @@ -3725,6 +3755,8 @@ pg_namespace_aclmask_ext(Oid nsp_oid, Oid roleid, ownerId = ((Form_pg_namespace) GETSTRUCT(tuple))->nspowner; + mdb_read_all_data_oid = get_role_oid("mdb_read_all_data", true); + aclDatum = SysCacheGetAttr(NAMESPACEOID, tuple, Anum_pg_namespace_nspacl, &isNull); if (isNull) @@ -3757,6 +3789,13 @@ pg_namespace_aclmask_ext(Oid nsp_oid, Oid roleid, (has_privs_of_role(roleid, ROLE_PG_READ_ALL_DATA) || has_privs_of_role(roleid, ROLE_PG_WRITE_ALL_DATA))) result |= ACL_USAGE; + + + if (mask & ACL_USAGE && !(result & ACL_USAGE) && + has_privs_of_role(roleid, mdb_read_all_data_oid) && + !has_privs_of_unwanted_system_role(ownerId, true)) + result |= ACL_USAGE; + return result; } diff --git a/src/test/regress/expected/mdb_read_write_roles.out b/src/test/regress/expected/mdb_read_write_roles.out new file mode 100644 index 00000000000..4c8b04eec89 --- /dev/null +++ b/src/test/regress/expected/mdb_read_write_roles.out @@ -0,0 +1,97 @@ +CREATE ROLE regress_mdb_superuser_user1; +CREATE ROLE regress_mdb_superuser_user2; +CREATE ROLE regress_mdb_superuser_user3; +CREATE ROLE regress_superuser WITH SUPERUSER; +GRANT mdb_superuser TO regress_mdb_superuser_user1; +GRANT CREATE ON DATABASE regression TO regress_mdb_superuser_user2; +SET ROLE regress_superuser; +CREATE TABLE regress_superuser_table(); +SET ROLE pg_read_server_files; +CREATE TABLE regress_pgrsf_table(); +SET ROLE pg_write_server_files; +CREATE TABLE regress_pgwsf_table(); +SET ROLE pg_execute_server_program; +CREATE TABLE regress_pgxsp_table(); +SET ROLE pg_read_all_data; +CREATE TABLE regress_pgrad_table(); +SET ROLE pg_write_all_data; +CREATE TABLE regress_pgrwd_table(); +SET ROLE regress_mdb_superuser_user1; +CREATE TABLE regress_mdbsu_table(); +CREATE SCHEMA regress_schema CREATE TABLE regress_mdbsu_table(); +SET ROLE regress_mdb_superuser_user2; +CREATE TABLE regress_mdbsu_table2(); +CREATE SCHEMA regress_schema2 CREATE TABLE regress_mdbsu_table2(); +SET ROLE mdb_read_all_data; +-- cannot read all data (fail) +TABLE pg_authid; +ERROR: permission denied for table pg_authid +TABLE regress_superuser_table; +ERROR: permission denied for table regress_superuser_table +TABLE regress_pgrsf_table; +ERROR: permission denied for table regress_pgrsf_table +TABLE regress_pgwsf_table; +ERROR: permission denied for table regress_pgwsf_table +TABLE regress_pgxsp_table; +ERROR: permission denied for table regress_pgxsp_table +TABLE regress_pgrad_table; +ERROR: permission denied for table regress_pgrad_table +TABLE regress_pgwsf_table; +ERROR: permission denied for table regress_pgwsf_table +-- is allow to read all other data +TABLE regress_mdbsu_table; +-- +(0 rows) + +TABLE regress_mdbsu_table2; +-- +(0 rows) + +-- check USAGE of schema +TABLE regress_schema.regress_mdbsu_table; +-- +(0 rows) + +TABLE regress_schema2.regress_mdbsu_table2; +-- +(0 rows) + +SET ROLE mdb_write_all_data; +CREATE TABLE regress_tt_dat(); +-- cannot read all data (fail) +INSERT INTO regress_superuser_table TABLE regress_tt_dat; +ERROR: permission denied for table regress_superuser_table +INSERT INTO regress_pgrsf_table TABLE regress_tt_dat; +ERROR: permission denied for table regress_pgrsf_table +INSERT INTO regress_pgwsf_table TABLE regress_tt_dat; +ERROR: permission denied for table regress_pgwsf_table +INSERT INTO regress_pgxsp_table TABLE regress_tt_dat; +ERROR: permission denied for table regress_pgxsp_table +INSERT INTO regress_pgrad_table TABLE regress_tt_dat; +ERROR: permission denied for table regress_pgrad_table +INSERT INTO regress_pgwsf_table TABLE regress_tt_dat; +ERROR: permission denied for table regress_pgwsf_table +-- is allow to read all other data +INSERT INTO regress_mdbsu_table TABLE regress_tt_dat; +INSERT INTO regress_mdbsu_table2 TABLE regress_tt_dat; +-- end tests +RESET SESSION AUTHORIZATION; +-- +DROP TABLE regress_pgrsf_table; +DROP TABLE regress_pgwsf_table; +DROP TABLE regress_pgxsp_table; +DROP TABLE regress_pgrad_table; +DROP TABLE regress_pgrwd_table; +DROP TABLE regress_mdbsu_table; +DROP TABLE regress_mdbsu_table2; +DROP TABLE regress_schema.regress_mdbsu_table; +DROP TABLE regress_schema2.regress_mdbsu_table2; +DROP SCHEMA regress_schema; +DROP SCHEMA regress_schema2; +DROP TABLE regress_superuser_table; +REVOKE CREATE ON DATABASE regression FROM regress_mdb_superuser_user2; +REVOKE CREATE ON DATABASE regression FROM regress_mdb_superuser_user3; +DROP ROLE regress_mdb_superuser_user1; +DROP ROLE regress_mdb_superuser_user2; +DROP ROLE regress_mdb_superuser_user3; +DROP ROLE regress_superuser; diff --git a/src/test/regress/expected/mdb_superuser.out b/src/test/regress/expected/mdb_superuser.out index 7849311cd1d..82de2e63461 100644 --- a/src/test/regress/expected/mdb_superuser.out +++ b/src/test/regress/expected/mdb_superuser.out @@ -4,7 +4,6 @@ CREATE ROLE regress_mdb_superuser_user3; CREATE ROLE regress_mdb_su_role_o1; CREATE ROLE regress_mdb_su_role_o2; GRANT mdb_superuser TO regress_mdb_su_role_o1; -GRANT mdb_admin TO mdb_superuser; CREATE ROLE regress_superuser WITH SUPERUSER; GRANT mdb_superuser TO regress_mdb_superuser_user1; GRANT CREATE ON DATABASE regression TO regress_mdb_superuser_user2; @@ -59,7 +58,7 @@ DROP TABLE regress_pgwsf_table; DROP TABLE regress_pgxsp_table; DROP TABLE regress_pgrad_table; DROP TABLE regress_pgrwd_table; --- does allowed to creare database, role or extension +-- does NOT allowed to create database, role or extension -- or grant such priviledge CREATE DATABASE regress_db_fail; ERROR: permission denied to create database @@ -128,3 +127,4 @@ DROP SCHEMA regress_mdb_superuser_schema; DROP ROLE regress_mdb_superuser_user1; DROP ROLE regress_mdb_superuser_user2; DROP ROLE regress_mdb_superuser_user3; +DROP ROLE regress_superuser; diff --git a/src/test/regress/expected/test_setup.out b/src/test/regress/expected/test_setup.out index 2344180f422..3b678093a14 100644 --- a/src/test/regress/expected/test_setup.out +++ b/src/test/regress/expected/test_setup.out @@ -241,5 +241,8 @@ create function fipshash(text) return substr(encode(sha256($1::bytea), 'hex'), 1, 32); CREATE ROLE mdb_admin; CREATE ROLE mdb_superuser; +CREATE ROLE mdb_read_all_data; +CREATE ROLE mdb_write_all_data; CREATE ROLE mdb_replication; +GRANT mdb_admin TO mdb_superuser; GRANT pg_create_subscription TO mdb_admin; diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule index 6a6574036c8..144bdbcc116 100644 --- a/src/test/regress/parallel_schedule +++ b/src/test/regress/parallel_schedule @@ -21,6 +21,8 @@ test: mdb_replication test: mdb_copy +test: mdb_read_write_roles + # ---------- # The first group of parallel tests # ---------- diff --git a/src/test/regress/sql/mdb_read_write_roles.sql b/src/test/regress/sql/mdb_read_write_roles.sql new file mode 100644 index 00000000000..0522e45a631 --- /dev/null +++ b/src/test/regress/sql/mdb_read_write_roles.sql @@ -0,0 +1,101 @@ +CREATE ROLE regress_mdb_superuser_user1; +CREATE ROLE regress_mdb_superuser_user2; +CREATE ROLE regress_mdb_superuser_user3; + +CREATE ROLE regress_superuser WITH SUPERUSER; + +GRANT mdb_superuser TO regress_mdb_superuser_user1; + +GRANT CREATE ON DATABASE regression TO regress_mdb_superuser_user2; + +SET ROLE regress_superuser; +CREATE TABLE regress_superuser_table(); + +SET ROLE pg_read_server_files; +CREATE TABLE regress_pgrsf_table(); + +SET ROLE pg_write_server_files; +CREATE TABLE regress_pgwsf_table(); + +SET ROLE pg_execute_server_program; +CREATE TABLE regress_pgxsp_table(); + +SET ROLE pg_read_all_data; +CREATE TABLE regress_pgrad_table(); + +SET ROLE pg_write_all_data; +CREATE TABLE regress_pgrwd_table(); + +SET ROLE regress_mdb_superuser_user1; +CREATE TABLE regress_mdbsu_table(); +CREATE SCHEMA regress_schema CREATE TABLE regress_mdbsu_table(); + +SET ROLE regress_mdb_superuser_user2; +CREATE TABLE regress_mdbsu_table2(); +CREATE SCHEMA regress_schema2 CREATE TABLE regress_mdbsu_table2(); + +SET ROLE mdb_read_all_data; +-- cannot read all data (fail) +TABLE pg_authid; +TABLE regress_superuser_table; +TABLE regress_pgrsf_table; +TABLE regress_pgwsf_table; +TABLE regress_pgxsp_table; +TABLE regress_pgrad_table; +TABLE regress_pgwsf_table; + + +-- is allow to read all other data + +TABLE regress_mdbsu_table; +TABLE regress_mdbsu_table2; + +-- check USAGE of schema +TABLE regress_schema.regress_mdbsu_table; +TABLE regress_schema2.regress_mdbsu_table2; + +SET ROLE mdb_write_all_data; +CREATE TABLE regress_tt_dat(); +-- cannot read all data (fail) +INSERT INTO regress_superuser_table TABLE regress_tt_dat; +INSERT INTO regress_pgrsf_table TABLE regress_tt_dat; +INSERT INTO regress_pgwsf_table TABLE regress_tt_dat; +INSERT INTO regress_pgxsp_table TABLE regress_tt_dat; +INSERT INTO regress_pgrad_table TABLE regress_tt_dat; +INSERT INTO regress_pgwsf_table TABLE regress_tt_dat; + + +-- is allow to read all other data + +INSERT INTO regress_mdbsu_table TABLE regress_tt_dat; +INSERT INTO regress_mdbsu_table2 TABLE regress_tt_dat; + +-- end tests + +RESET SESSION AUTHORIZATION; +-- + +DROP TABLE regress_pgrsf_table; +DROP TABLE regress_pgwsf_table; +DROP TABLE regress_pgxsp_table; +DROP TABLE regress_pgrad_table; +DROP TABLE regress_pgrwd_table; + +DROP TABLE regress_mdbsu_table; +DROP TABLE regress_mdbsu_table2; + +DROP TABLE regress_schema.regress_mdbsu_table; +DROP TABLE regress_schema2.regress_mdbsu_table2; + +DROP SCHEMA regress_schema; +DROP SCHEMA regress_schema2; + +DROP TABLE regress_superuser_table; + +REVOKE CREATE ON DATABASE regression FROM regress_mdb_superuser_user2; +REVOKE CREATE ON DATABASE regression FROM regress_mdb_superuser_user3; + +DROP ROLE regress_mdb_superuser_user1; +DROP ROLE regress_mdb_superuser_user2; +DROP ROLE regress_mdb_superuser_user3; +DROP ROLE regress_superuser; diff --git a/src/test/regress/sql/mdb_superuser.sql b/src/test/regress/sql/mdb_superuser.sql index 0877d9258f6..cc5caf531b3 100644 --- a/src/test/regress/sql/mdb_superuser.sql +++ b/src/test/regress/sql/mdb_superuser.sql @@ -7,8 +7,6 @@ CREATE ROLE regress_mdb_su_role_o2; GRANT mdb_superuser TO regress_mdb_su_role_o1; -GRANT mdb_admin TO mdb_superuser; - CREATE ROLE regress_superuser WITH SUPERUSER; GRANT mdb_superuser TO regress_mdb_superuser_user1; @@ -90,7 +88,7 @@ DROP TABLE regress_pgrad_table; DROP TABLE regress_pgrwd_table; --- does allowed to creare database, role or extension +-- does NOT allowed to create database, role or extension -- or grant such priviledge CREATE DATABASE regress_db_fail; @@ -162,3 +160,4 @@ DROP SCHEMA regress_mdb_superuser_schema; DROP ROLE regress_mdb_superuser_user1; DROP ROLE regress_mdb_superuser_user2; DROP ROLE regress_mdb_superuser_user3; +DROP ROLE regress_superuser; diff --git a/src/test/regress/sql/test_setup.sql b/src/test/regress/sql/test_setup.sql index ef096c6b930..c71d53d87e0 100644 --- a/src/test/regress/sql/test_setup.sql +++ b/src/test/regress/sql/test_setup.sql @@ -297,6 +297,9 @@ create function fipshash(text) CREATE ROLE mdb_admin; CREATE ROLE mdb_superuser; +CREATE ROLE mdb_read_all_data; +CREATE ROLE mdb_write_all_data; CREATE ROLE mdb_replication; +GRANT mdb_admin TO mdb_superuser; GRANT pg_create_subscription TO mdb_admin; From 7e48f7826a3aabb7e2b9ec55c3584d9c8910ad92 Mon Sep 17 00:00:00 2001 From: reshke Date: Sat, 26 Apr 2025 08:16:33 +0000 Subject: [PATCH 43/54] Add check for mdb_service_auth role --- src/backend/catalog/aclchk.c | 4 ++-- src/backend/utils/adt/acl.c | 20 +++++++++++++++----- src/include/utils/acl.h | 2 +- 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/src/backend/catalog/aclchk.c b/src/backend/catalog/aclchk.c index eb1c2cf663d..9a9cd32200c 100644 --- a/src/backend/catalog/aclchk.c +++ b/src/backend/catalog/aclchk.c @@ -3448,7 +3448,7 @@ pg_class_aclmask_ext(Oid table_oid, Oid roleid, AclMode mask, */ if (mask & ACL_SELECT && !(result & ACL_SELECT) && has_privs_of_role(roleid, mdb_read_all_data_oid) && - !has_privs_of_unwanted_system_role(ownerId)) + !has_privs_of_unwanted_system_role(ownerId, true)) result |= ACL_SELECT; /* @@ -3473,7 +3473,7 @@ pg_class_aclmask_ext(Oid table_oid, Oid roleid, AclMode mask, if (mask & (ACL_INSERT | ACL_UPDATE | ACL_DELETE) && !(result & (ACL_INSERT | ACL_UPDATE | ACL_DELETE)) && has_privs_of_role(roleid, mdb_write_all_data_oid) && - !has_privs_of_unwanted_system_role(ownerId)) + !has_privs_of_unwanted_system_role(ownerId, true)) result |= (mask & (ACL_INSERT | ACL_UPDATE | ACL_DELETE)); /* diff --git a/src/backend/utils/adt/acl.c b/src/backend/utils/adt/acl.c index 8d71e5b9a91..19909a9ea5f 100644 --- a/src/backend/utils/adt/acl.c +++ b/src/backend/utils/adt/acl.c @@ -5203,7 +5203,9 @@ has_privs_of_role_strict_no_cache(Oid member, Oid role) */ bool -has_privs_of_unwanted_system_role(Oid role) { +has_privs_of_unwanted_system_role(Oid role, bool check_mdb_service_auth) { + Oid mdb_service_authoid; + if (has_privs_of_role_strict(role, ROLE_PG_READ_SERVER_FILES)) { return true; } @@ -5220,6 +5222,14 @@ has_privs_of_unwanted_system_role(Oid role) { return true; } + if (check_mdb_service_auth) { + mdb_service_authoid = get_role_oid("mdb_service_auth", true); + + if (has_privs_of_role_strict(role, mdb_service_authoid)) { + return true; + } + } + return false; } @@ -5266,7 +5276,7 @@ has_privs_of_role(Oid member, Oid role) * if target role is neither superuser nor * some dangerous system role */ - if (!has_privs_of_unwanted_system_role(role)) { + if (!has_privs_of_unwanted_system_role(role, true)) { return true; } } @@ -5321,7 +5331,7 @@ mdb_admin_allow_bypass_owner_checks(Oid userId, Oid ownerId) */ /* All checks passed, hope will not be hacked here (again) */ - return !has_privs_of_unwanted_system_role(ownerId); + return !has_privs_of_unwanted_system_role(ownerId, true); } // -- non-upstream patch end @@ -5400,7 +5410,7 @@ check_mdb_admin_is_member_of_role(Oid member, Oid role) GetUserNameFromId(role, false)))); } - if (has_privs_of_unwanted_system_role(role)) { + if (has_privs_of_unwanted_system_role(role, true)) { ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("forbidden to transfer ownership to this system role in Cloud"))); @@ -5434,7 +5444,7 @@ bool mdb_admin_is_member_of_role(Oid member, Oid role) { return false; } - return !has_privs_of_unwanted_system_role(role); + return !has_privs_of_unwanted_system_role(role, true); } // -- mdb admin patch diff --git a/src/include/utils/acl.h b/src/include/utils/acl.h index 38011873bf3..e48774d89e8 100644 --- a/src/include/utils/acl.h +++ b/src/include/utils/acl.h @@ -231,7 +231,7 @@ extern bool mdb_admin_is_member_of_role(Oid member, Oid role); extern Oid select_best_admin(Oid member, Oid role); extern Oid get_role_oid(const char *rolname, bool missing_ok); extern Oid get_role_oid_or_public(const char *rolname); -extern bool has_privs_of_unwanted_system_role(Oid role); +extern bool has_privs_of_unwanted_system_role(Oid role, bool check_mdb_service_auth); extern Oid get_rolespec_oid(const RoleSpec *role, bool missing_ok); extern void check_rolespec_name(const RoleSpec *role, const char *detail_msg); extern HeapTuple get_rolespec_tuple(const RoleSpec *role); From 60d956981a3b2941ff810c09181d8b11d7de1300 Mon Sep 17 00:00:00 2001 From: reshke Date: Fri, 15 Aug 2025 14:16:02 +0000 Subject: [PATCH 44/54] Make mdb_admin and mdb_superuser an implicit grant with pg_signal_autovacuum Until pg18 we have to work with our internal implementation of pg_signal_autovacuum_worker. Since explicit GRANT will cause problems during pg_upgrade, use internal implicit grant logic --- src/backend/utils/adt/acl.c | 14 ++++++++++++++ src/test/mdb_admin/t/signals.pl | 1 - src/test/regress/expected/mdb_admin.out | 12 ++++++++++++ src/test/regress/sql/mdb_admin.sql | 3 +++ 4 files changed, 29 insertions(+), 1 deletion(-) diff --git a/src/backend/utils/adt/acl.c b/src/backend/utils/adt/acl.c index 19909a9ea5f..22bfc638675 100644 --- a/src/backend/utils/adt/acl.c +++ b/src/backend/utils/adt/acl.c @@ -5257,6 +5257,7 @@ bool has_privs_of_role(Oid member, Oid role) { Oid mdb_superuser_roleoid; + Oid mdb_admin_roleoid; /* Fast path for simple case */ if (member == role) @@ -5269,6 +5270,10 @@ has_privs_of_role(Oid member, Oid role) mdb_superuser_roleoid = get_role_oid("mdb_superuser", true /*if nodoby created mdb_superuser role in this database*/); if (is_member_of_role(member, mdb_superuser_roleoid)) { + /* mdb superuser is member of role pg_signal_autovacuum */ + if (role == ROLE_PG_SIGNAL_AUTOVACUUM) { + return true; + } /* if target role is superuser, disallow */ if (!superuser_arg(role)) { /* we want mdb_roles_admin to bypass @@ -5281,6 +5286,15 @@ has_privs_of_role(Oid member, Oid role) } } } + + mdb_admin_roleoid = get_role_oid("mdb_admin", true /*if nodoby created mdb_superuser role in this database*/); + + if (is_member_of_role(member, mdb_admin_roleoid)) { + /* mdb admin is member of role pg_signal_autovacuum */ + if (role == ROLE_PG_SIGNAL_AUTOVACUUM) { + return true; + } + } /* diff --git a/src/test/mdb_admin/t/signals.pl b/src/test/mdb_admin/t/signals.pl index cb9c2e12db2..a4b3f07fb89 100644 --- a/src/test/mdb_admin/t/signals.pl +++ b/src/test/mdb_admin/t/signals.pl @@ -18,7 +18,6 @@ " CREATE DATABASE regress; CREATE ROLE mdb_admin; - GRANT pg_signal_autovacuum TO mdb_admin; CREATE ROLE mdb_reg_lh_1; CREATE ROLE mdb_reg_lh_2; GRANT pg_signal_backend TO mdb_admin; diff --git a/src/test/regress/expected/mdb_admin.out b/src/test/regress/expected/mdb_admin.out index 20722750c05..90f76813cf5 100644 --- a/src/test/regress/expected/mdb_admin.out +++ b/src/test/regress/expected/mdb_admin.out @@ -84,6 +84,18 @@ ALTER TABLE regress_mdb_admin_table OWNER TO pg_read_all_data; ERROR: must be able to SET ROLE "pg_read_all_data" ALTER SCHEMA regress_mdb_admin_schema OWNER TO pg_read_all_data; ERROR: must be able to SET ROLE "pg_read_all_data" +SELECT pg_has_role('mdb_admin', 'pg_signal_autovacuum', 'usage'); + pg_has_role +------------- + t +(1 row) + +SELECT pg_has_role('mdb_superuser', 'pg_signal_autovacuum', 'usage'); + pg_has_role +------------- + t +(1 row) + -- end tests RESET SESSION AUTHORIZATION; -- diff --git a/src/test/regress/sql/mdb_admin.sql b/src/test/regress/sql/mdb_admin.sql index b6b048e5692..14e96ec1d70 100644 --- a/src/test/regress/sql/mdb_admin.sql +++ b/src/test/regress/sql/mdb_admin.sql @@ -69,6 +69,9 @@ ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO pg_read_al ALTER TABLE regress_mdb_admin_table OWNER TO pg_read_all_data; ALTER SCHEMA regress_mdb_admin_schema OWNER TO pg_read_all_data; +SELECT pg_has_role('mdb_admin', 'pg_signal_autovacuum', 'usage'); +SELECT pg_has_role('mdb_superuser', 'pg_signal_autovacuum', 'usage'); + -- end tests RESET SESSION AUTHORIZATION; From 8330f7cdbe04f1db4decc5eb1a332f205dc4b284 Mon Sep 17 00:00:00 2001 From: rkhapov Date: Mon, 12 May 2025 15:29:17 +0500 Subject: [PATCH 45/54] pg_stat_statements.c: cancelable qtext_load_file In the case of a large PGSS_TEXT_FILE, the work time of the qtext_load_file function will be quite long, and the query to the pg_stat_statements table will not be cancellable, as there is no CHECK_FOR_INTERRUPT in the function. Also, the amount of bytes read can reach 1 GB, which leads to a slow read system call that does not allow cancellation of the query. Testing the speed of sequential read using fio with different block sizes shows that there is no significant difference between 16 MB blocks and 1 GB blocks. Therefore, this patch changes the maximum read value from 1 GB to 16 MB and adds INTERRUPTS_PENDING_CONDITION() check in the read loop of qtext_load_file to make it cancellable. For now, only statement execution is cancellable (fail_on_interrupt is true only for calls from pg_stat_statements_internal) Signed-off-by: rkhapov Reviewed-by: reshke --- .../pg_stat_statements/pg_stat_statements.c | 28 ++++++++++++++----- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/contrib/pg_stat_statements/pg_stat_statements.c b/contrib/pg_stat_statements/pg_stat_statements.c index 82a4141aed7..23c055daf30 100644 --- a/contrib/pg_stat_statements/pg_stat_statements.c +++ b/contrib/pg_stat_statements/pg_stat_statements.c @@ -358,7 +358,7 @@ static pgssEntry *entry_alloc(pgssHashKey *key, Size query_offset, int query_len static void entry_dealloc(void); static bool qtext_store(const char *query, int query_len, Size *query_offset, int *gc_count); -static char *qtext_load_file(Size *buffer_size); +static char *qtext_load_file(Size *buffer_size, bool fail_on_interrupts); static char *qtext_fetch(Size query_offset, int query_len, char *buffer, Size buffer_size); static bool need_gc_qtexts(void); @@ -760,7 +760,7 @@ pgss_shmem_shutdown(int code, Datum arg) if (fwrite(&num_entries, sizeof(int32), 1, file) != 1) goto error; - qbuffer = qtext_load_file(&qbuffer_size); + qbuffer = qtext_load_file(&qbuffer_size, false /* fail_on_interrupts */ ); if (qbuffer == NULL) goto error; @@ -1742,7 +1742,7 @@ pg_stat_statements_internal(FunctionCallInfo fcinfo, /* No point in loading file now if there are active writers */ if (n_writers == 0) - qbuffer = qtext_load_file(&qbuffer_size); + qbuffer = qtext_load_file(&qbuffer_size, true /* fail_on_interrupts */ ); } /* @@ -1776,7 +1776,7 @@ pg_stat_statements_internal(FunctionCallInfo fcinfo, { if (qbuffer) pfree(qbuffer); - qbuffer = qtext_load_file(&qbuffer_size); + qbuffer = qtext_load_file(&qbuffer_size, true /* fail_on_interrupts */ ); } } @@ -1795,6 +1795,12 @@ pg_stat_statements_internal(FunctionCallInfo fcinfo, memset(values, 0, sizeof(values)); memset(nulls, 0, sizeof(nulls)); + /* Can't process interrupts here - pgss-lock is acquired */ + if (INTERRUPTS_PENDING_CONDITION()) + { + break; + } + values[i++] = ObjectIdGetDatum(entry->key.userid); values[i++] = ObjectIdGetDatum(entry->key.dbid); if (api_version >= PGSS_V1_9) @@ -1978,6 +1984,7 @@ pg_stat_statements_internal(FunctionCallInfo fcinfo, if (qbuffer) pfree(qbuffer); + CHECK_FOR_INTERRUPTS(); } /* Number of output arguments (columns) for pg_stat_statements_info */ @@ -2295,7 +2302,7 @@ qtext_store(const char *query, int query_len, * the caller is responsible for verifying that the result is sane. */ static char * -qtext_load_file(Size *buffer_size) +qtext_load_file(Size *buffer_size, bool fail_on_interrupts) { char *buf; int fd; @@ -2348,7 +2355,14 @@ qtext_load_file(Size *buffer_size) nread = 0; while (nread < stat.st_size) { - int toread = Min(1024 * 1024 * 1024, stat.st_size - nread); + int toread = Min(32 * 1024 * 1024, stat.st_size - nread); + + if (fail_on_interrupts && INTERRUPTS_PENDING_CONDITION()) + { + free(buf); + CloseTransientFile(fd); + return NULL; + } /* * If we get a short read and errno doesn't get set, the reason is @@ -2489,7 +2503,7 @@ gc_qtexts(void) * file is only going to get bigger; hoping for a future non-OOM result is * risky and can easily lead to complete denial of service. */ - qbuffer = qtext_load_file(&qbuffer_size); + qbuffer = qtext_load_file(&qbuffer_size, false /* fail_on_interrupts */ ); if (qbuffer == NULL) goto gc_fail; From 1fdd8536d4345fb846d525ebb9761539b26d2b9e Mon Sep 17 00:00:00 2001 From: Victor Popov Date: Mon, 1 Dec 2025 17:24:19 +0300 Subject: [PATCH 46/54] MDB-40410: Allow to kill backends which have application_name starting with "MDB" instead of exactly matching --- src/backend/storage/ipc/signalfuncs.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/storage/ipc/signalfuncs.c b/src/backend/storage/ipc/signalfuncs.c index 5fc16d35d0c..7d7af85559a 100644 --- a/src/backend/storage/ipc/signalfuncs.c +++ b/src/backend/storage/ipc/signalfuncs.c @@ -96,7 +96,7 @@ pg_signal_backend(int pid, int sig) if (!((local_beentry && local_beentry->backendStatus.st_backendType == B_AUTOVAC_WORKER && has_privs_of_role(GetUserId(), ROLE_PG_SIGNAL_AUTOVACUUM)) - || (appname != NULL && strcmp(appname, "MDB") == 0))) + || (appname != NULL && strncmp(appname, "MDB", 3) == 0))) return SIGNAL_BACKEND_NOSUPERUSER; } else { if (superuser_arg(proc->roleId)) From 68b4b5de37ef4f0030d2b7e230a35b2ad69180a1 Mon Sep 17 00:00:00 2001 From: reshke Date: Mon, 8 Dec 2025 08:02:06 +0000 Subject: [PATCH 47/54] Fix MDB-signal process and add TAP test for feature --- src/backend/storage/ipc/signalfuncs.c | 20 ++++--- src/test/mdb_admin/t/mdb_app_name.pl | 75 +++++++++++++++++++++++++++ 2 files changed, 89 insertions(+), 6 deletions(-) create mode 100644 src/test/mdb_admin/t/mdb_app_name.pl diff --git a/src/backend/storage/ipc/signalfuncs.c b/src/backend/storage/ipc/signalfuncs.c index 7d7af85559a..caec35ae42b 100644 --- a/src/backend/storage/ipc/signalfuncs.c +++ b/src/backend/storage/ipc/signalfuncs.c @@ -87,6 +87,13 @@ pg_signal_backend(int pid, int sig) */ if (!superuser()) { if (!OidIsValid(proc->roleId)) { + LocalPgBackendStatus *local_beentry; + local_beentry = pgstat_get_local_beentry_by_proc_number(GetNumberFromPGProc(proc)); + + if (!(local_beentry && local_beentry->backendStatus.st_backendType == B_AUTOVAC_WORKER && + has_privs_of_role(GetUserId(), ROLE_PG_SIGNAL_AUTOVACUUM))) + return SIGNAL_BACKEND_NOSUPERUSER; + } else { LocalPgBackendStatus *local_beentry; char * appname = NULL; local_beentry = pgstat_get_local_beentry_by_proc_number(GetNumberFromPGProc(proc)); @@ -94,12 +101,13 @@ pg_signal_backend(int pid, int sig) appname = local_beentry->backendStatus.st_appname; } - if (!((local_beentry && local_beentry->backendStatus.st_backendType == B_AUTOVAC_WORKER && - has_privs_of_role(GetUserId(), ROLE_PG_SIGNAL_AUTOVACUUM)) - || (appname != NULL && strncmp(appname, "MDB", 3) == 0))) - return SIGNAL_BACKEND_NOSUPERUSER; - } else { - if (superuser_arg(proc->roleId)) + /* NB: always upper-case */ + if (appname != NULL && strncmp(appname, "MDB", 3) == 0) + { + /* ok, allow, but still re-check for ROLE_PG_SIGNAL_BACKEND */ + /* this code is written like that for sake of rebase, do not change codestyle */ + } + else if (superuser_arg(proc->roleId)) return SIGNAL_BACKEND_NOSUPERUSER; /* Users can signal backends they have role membership in. */ diff --git a/src/test/mdb_admin/t/mdb_app_name.pl b/src/test/mdb_admin/t/mdb_app_name.pl new file mode 100644 index 00000000000..a040e5ec4a9 --- /dev/null +++ b/src/test/mdb_admin/t/mdb_app_name.pl @@ -0,0 +1,75 @@ + +# Copyright (c) 2024-2024, MDB, Mother Russia + +# Minimal test testing streaming replication +use strict; +use warnings; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +# Initialize primary node +my $node_primary = PostgreSQL::Test::Cluster->new('primary'); +$node_primary->init(); +$node_primary->start; + +# Create some content on primary and check its presence in standby nodes +$node_primary->safe_psql('postgres', + " + CREATE DATABASE regress; + CREATE ROLE mdb_reg_lh_app_name; + GRANT pg_signal_backend to mdb_reg_lh_app_name; + CREATE TABLE mdb_app_name_t(i int); +"); + +my $main_sess = $node_primary->background_psql('postgres'); + +$main_sess->query_safe( + q( +SET application_name TO 'SU'; +BEGIN; +INSERT INTO mdb_app_name_t VALUES(0); +)); + + +my $res_pid = $node_primary->safe_psql('regress', + " + SELECT pid FROM pg_stat_activity WHERE application_name = 'SU'; +"); + +print "pid is $res_pid\n"; + +ok(1); + + +my ($res_reg_lh_1, $stdout_reg_lh_1, $stderr_reg_lh_1) = $node_primary->psql('regress', + " + SET ROLE mdb_reg_lh_app_name; + SELECT pg_terminate_backend($res_pid); +"); + +# print ($res_reg_lh_1, $stdout_reg_lh_1, $stderr_reg_lh_1, "\n"); + +ok($res_reg_lh_1 != 0, "should fail for non-MDB"); +like($stderr_reg_lh_1, qr/Only roles with the SUPERUSER attribute may terminate processes of roles with the SUPERUSER attribute./, "matches"); + +# should succeed +$main_sess->query_safe(qq[COMMIT]); + +$main_sess->query_safe( + q( +SET application_name TO 'MDB'; +BEGIN; +INSERT INTO mdb_app_name_t VALUES(1); +)); + + +my ($res_reg_lh_2, $stdout_reg_lh_2, $stderr_reg_lh_2) = $node_primary->psql('regress', + " + SET ROLE mdb_reg_lh_app_name; + SELECT pg_terminate_backend($res_pid); +"); + +ok($res_reg_lh_2 == 0, "should success for MDB"); + +done_testing(); From 6b57aea44575e11fa4b19a6416d2baae5b9c08ed Mon Sep 17 00:00:00 2001 From: reshke Date: Sun, 18 Jan 2026 14:12:21 +0000 Subject: [PATCH 48/54] v6 of bt_page_items pretty-print --- contrib/pageinspect/Makefile | 3 +- contrib/pageinspect/btreefuncs.c | 142 ++++++++++++++++-- contrib/pageinspect/expected/btree.out | 25 +++ .../pageinspect/pageinspect--1.12--1.13.sql | 21 +++ contrib/pageinspect/pageinspect.control | 2 +- contrib/pageinspect/sql/btree.sql | 4 + 6 files changed, 185 insertions(+), 12 deletions(-) create mode 100644 contrib/pageinspect/pageinspect--1.12--1.13.sql diff --git a/contrib/pageinspect/Makefile b/contrib/pageinspect/Makefile index 95e030b3969..9dee7653310 100644 --- a/contrib/pageinspect/Makefile +++ b/contrib/pageinspect/Makefile @@ -13,7 +13,8 @@ OBJS = \ rawpage.o EXTENSION = pageinspect -DATA = pageinspect--1.11--1.12.sql pageinspect--1.10--1.11.sql \ +DATA = pageinspect--1.12--1.13.sql \ + pageinspect--1.11--1.12.sql pageinspect--1.10--1.11.sql \ pageinspect--1.9--1.10.sql pageinspect--1.8--1.9.sql \ pageinspect--1.7--1.8.sql pageinspect--1.6--1.7.sql \ pageinspect--1.5.sql pageinspect--1.5--1.6.sql \ diff --git a/contrib/pageinspect/btreefuncs.c b/contrib/pageinspect/btreefuncs.c index 9cdc8e182b4..94dd8641686 100644 --- a/contrib/pageinspect/btreefuncs.c +++ b/contrib/pageinspect/btreefuncs.c @@ -29,6 +29,7 @@ #include "access/nbtree.h" #include "access/relation.h" +#include "access/tupdesc.h" #include "catalog/namespace.h" #include "catalog/pg_am.h" #include "catalog/pg_type.h" @@ -38,6 +39,8 @@ #include "utils/array.h" #include "utils/builtins.h" #include "utils/rel.h" +#include "utils/ruleutils.h" +#include "utils/lsyscache.h" #include "utils/varlena.h" PG_FUNCTION_INFO_V1(bt_metap); @@ -95,6 +98,8 @@ typedef struct ua_page_items bool leafpage; bool rightmost; TupleDesc tupd; + Relation indexRel; + bool pretty_print; } ua_page_items; @@ -548,17 +553,118 @@ bt_page_print_tuples(ua_page_items *uargs) if (dlen < 0 || dlen > INDEX_SIZE_MASK) elog(ERROR, "invalid tuple length %d for tuple at offset number %u", dlen, offset); - dump = palloc0(dlen * 3 + 1); - datacstring = dump; - for (off = 0; off < dlen; off++) + + if (!uargs->pretty_print) { - if (off > 0) - *dump++ = ' '; - sprintf(dump, "%02x", *(ptr + off) & 0xff); - dump += 2; + /* Old-style, print hex bytes */ + dump = palloc0(dlen * 3 + 1); + datacstring = dump; + for (off = 0; off < dlen; off++) + { + if (off > 0) + *dump++ = ' '; + sprintf(dump, "%02x", *(ptr + off) & 0xff); + dump += 2; + } + values[j++] = CStringGetTextDatum(datacstring); + pfree(datacstring); + } + else + { + /* Do pretty-print, akin to record_out() */ + StringInfoData buf; + TupleDesc tupdesc; + + Datum itup_values[INDEX_MAX_KEYS]; + bool itup_isnull[INDEX_MAX_KEYS]; + char *index_columns; + + /* + * Included attributes are added when dealing with leaf pages, discarded + * for non-leaf pages as these include only data for key attributes. + */ + int printflags = RULE_INDEXDEF_PRETTY; + if (P_ISLEAF((BTPageOpaque) PageGetSpecialPointer(page))) + { + tupdesc = RelationGetDescr(uargs->indexRel); + } + else + { + tupdesc = CreateTupleDescCopy(RelationGetDescr(uargs->indexRel)); + tupdesc->natts = IndexRelationGetNumberOfKeyAttributes(uargs->indexRel); + printflags |= RULE_INDEXDEF_KEYS_ONLY; + } + + index_columns = pg_get_indexdef_columns_extended(RelationGetRelid(uargs->indexRel), + printflags); + + + index_deform_tuple(itup, tupdesc, + itup_values, itup_isnull); + + + initStringInfo(&buf); + appendStringInfo(&buf, "(%s)=(", index_columns); + + for (int i = 0; i < tupdesc->natts; i++) + { + char *value; + char *tmp; + bool nq = false; + + if (itup_isnull[i]) + value = "null"; + else + { + Oid foutoid; + bool typisvarlena; + Oid typoid; + + typoid = TupleDescAttr(tupdesc, i)->atttypid; + getTypeOutputInfo(typoid, &foutoid, &typisvarlena); + value = OidOutputFunctionCall(foutoid, itup_values[i]); + } + + if (i == IndexRelationGetNumberOfKeyAttributes(uargs->indexRel)) + appendStringInfoString(&buf, ") INCLUDE ("); + else if (i > 0) + appendStringInfoString(&buf, ", "); + + /* Check whether we need double quotes for this value */ + nq = (value[0] == '\0'); /* force quotes for empty string */ + for (tmp = value; *tmp; tmp++) + { + char ch = *tmp; + + if (ch == '"' || ch == '\\' || + ch == '(' || ch == ')' || ch == ',' || + isspace((unsigned char) ch)) + { + nq = true; + break; + } + } + + /* And emit the string */ + if (nq) + appendStringInfoCharMacro(&buf, '"'); + for (tmp = value; *tmp; tmp++) + { + char ch = *tmp; + + if (ch == '"' || ch == '\\') + appendStringInfoCharMacro(&buf, ch); + appendStringInfoCharMacro(&buf, ch); + } + if (nq) + appendStringInfoCharMacro(&buf, '"'); + } + + appendStringInfoChar(&buf, ')'); + + values[j++] = CStringGetTextDatum(buf.data); + pfree(buf.data); } - values[j++] = CStringGetTextDatum(datacstring); - pfree(datacstring); /* * We need to work around the BTreeTupleIsPivot() !heapkeyspace limitation @@ -630,6 +736,11 @@ bt_page_items_internal(PG_FUNCTION_ARGS, enum pageinspect_version ext_version) FuncCallContext *fctx; MemoryContext mctx; ua_page_items *uargs; + bool pretty_print = false; + + if (PG_NARGS() >= 3) { + pretty_print = PG_GETARG_BOOL(2); + } if (!superuser()) ereport(ERROR, @@ -667,7 +778,6 @@ bt_page_items_internal(PG_FUNCTION_ARGS, enum pageinspect_version ext_version) memcpy(uargs->page, BufferGetPage(buffer), BLCKSZ); UnlockReleaseBuffer(buffer); - relation_close(rel, AccessShareLock); uargs->offset = FirstOffsetNumber; @@ -683,6 +793,8 @@ bt_page_items_internal(PG_FUNCTION_ARGS, enum pageinspect_version ext_version) } uargs->leafpage = P_ISLEAF(opaque); uargs->rightmost = P_RIGHTMOST(opaque); + uargs->pretty_print = pretty_print; + uargs->indexRel = rel; /* Build a tuple descriptor for our result type */ if (get_call_result_type(fcinfo, NULL, &tupleDesc) != TYPEFUNC_COMPOSITE) @@ -706,6 +818,9 @@ bt_page_items_internal(PG_FUNCTION_ARGS, enum pageinspect_version ext_version) SRF_RETURN_NEXT(fctx, result); } + if (uargs->indexRel) + relation_close(uargs->indexRel, AccessShareLock); + SRF_RETURN_DONE(fctx); } @@ -800,6 +915,10 @@ bt_page_items_bytea(PG_FUNCTION_ARGS) uargs->leafpage = P_ISLEAF(opaque); uargs->rightmost = P_RIGHTMOST(opaque); + uargs->pretty_print = false; + /* For bytea function, we cannot do pretty-print */ + uargs->indexRel = NULL; + /* Build a tuple descriptor for our result type */ if (get_call_result_type(fcinfo, NULL, &tupleDesc) != TYPEFUNC_COMPOSITE) elog(ERROR, "return type must be a row type"); @@ -822,6 +941,9 @@ bt_page_items_bytea(PG_FUNCTION_ARGS) SRF_RETURN_NEXT(fctx, result); } + if (uargs->indexRel) + relation_close(uargs->indexRel, AccessShareLock); + SRF_RETURN_DONE(fctx); } diff --git a/contrib/pageinspect/expected/btree.out b/contrib/pageinspect/expected/btree.out index 0aa5d73322f..f407a17121b 100644 --- a/contrib/pageinspect/expected/btree.out +++ b/contrib/pageinspect/expected/btree.out @@ -1,6 +1,7 @@ CREATE TABLE test1 (a int8, b int4range); INSERT INTO test1 VALUES (72057594037927937, '[0,1)'); CREATE INDEX test1_a_idx ON test1 USING btree (a); +CREATE INDEX test1_a_b_idx ON test1 USING btree (a, b); \x SELECT * FROM bt_metap('test1_a_idx'); -[ RECORD 1 ]-------------+------- @@ -147,6 +148,30 @@ btpo_flags | 1 SELECT * FROM bt_multi_page_stats('test2_col1_idx', 7, 2); ERROR: block number 7 is out of range DROP TABLE test2; +SELECT * FROM bt_page_items('test1_a_idx', 1, true); +-[ RECORD 1 ]----------------------- +itemoffset | 1 +ctid | (0,1) +itemlen | 16 +nulls | f +vars | f +data | (a)=(72057594037927937) +dead | f +htid | (0,1) +tids | + +SELECT * FROM bt_page_items('test1_a_b_idx', 1, true); +-[ RECORD 1 ]----------------------------------- +itemoffset | 1 +ctid | (0,1) +itemlen | 32 +nulls | f +vars | t +data | (a, b)=(72057594037927937, "[0,1)") +dead | f +htid | (0,1) +tids | + SELECT * FROM bt_page_items('test1_a_idx', -1); ERROR: invalid block number -1 SELECT * FROM bt_page_items('test1_a_idx', 0); diff --git a/contrib/pageinspect/pageinspect--1.12--1.13.sql b/contrib/pageinspect/pageinspect--1.12--1.13.sql new file mode 100644 index 00000000000..9917d13ba83 --- /dev/null +++ b/contrib/pageinspect/pageinspect--1.12--1.13.sql @@ -0,0 +1,21 @@ +/* contrib/pageinspect/pageinspect--1.12--1.13.sql */ + +-- complain if script is sourced in psql, rather than via ALTER EXTENSION +\echo Use "ALTER EXTENSION pageinspect UPDATE TO '1.13'" to load this file. \quit + + +DROP FUNCTION bt_page_items(text, int8); +CREATE FUNCTION bt_page_items(IN relname text, IN blkno int8, + IN pretty_print boolean DEFAULT FALSE, + OUT itemoffset smallint, + OUT ctid tid, + OUT itemlen smallint, + OUT nulls bool, + OUT vars bool, + OUT data text, + OUT dead boolean, + OUT htid tid, + OUT tids tid[]) +RETURNS SETOF record +AS 'MODULE_PATHNAME', 'bt_page_items_1_9' +LANGUAGE C STRICT PARALLEL SAFE; diff --git a/contrib/pageinspect/pageinspect.control b/contrib/pageinspect/pageinspect.control index b2804e9b128..cfc87feac03 100644 --- a/contrib/pageinspect/pageinspect.control +++ b/contrib/pageinspect/pageinspect.control @@ -1,5 +1,5 @@ # pageinspect extension comment = 'inspect the contents of database pages at a low level' -default_version = '1.12' +default_version = '1.13' module_pathname = '$libdir/pageinspect' relocatable = true diff --git a/contrib/pageinspect/sql/btree.sql b/contrib/pageinspect/sql/btree.sql index 102ebdefe3c..36f100656a4 100644 --- a/contrib/pageinspect/sql/btree.sql +++ b/contrib/pageinspect/sql/btree.sql @@ -1,6 +1,7 @@ CREATE TABLE test1 (a int8, b int4range); INSERT INTO test1 VALUES (72057594037927937, '[0,1)'); CREATE INDEX test1_a_idx ON test1 USING btree (a); +CREATE INDEX test1_a_b_idx ON test1 USING btree (a, b); \x @@ -22,6 +23,9 @@ SELECT * FROM bt_multi_page_stats('test2_col1_idx', 3, 2); SELECT * FROM bt_multi_page_stats('test2_col1_idx', 7, 2); DROP TABLE test2; +SELECT * FROM bt_page_items('test1_a_idx', 1, true); +SELECT * FROM bt_page_items('test1_a_b_idx', 1, true); + SELECT * FROM bt_page_items('test1_a_idx', -1); SELECT * FROM bt_page_items('test1_a_idx', 0); SELECT * FROM bt_page_items('test1_a_idx', 1); From e7d700d4a809f7c80db9511715aa66a55696586b Mon Sep 17 00:00:00 2001 From: Andrey Borodin Date: Tue, 10 Feb 2026 12:47:32 +0500 Subject: [PATCH 49/54] Add archive_mode=shared for coordinated WAL archiving Introduce a new archive_mode setting "shared" to prevent WAL history loss during standby promotion in HA streaming replication setups. In shared mode, the primary proactively sends archival status updates to standbys via the replication protocol. The standby creates .ready files for received WAL segments but defers marking them as .done until the primary confirms archival. This prevents WAL from being recycled before it's safely archived, addressing a critical gap in PITR continuity during failover. Key implementation details: - Primary periodically sends last archived WAL segment via new PqReplMsg_ArchiveStatusReport ('a') message - Standby marks all segments <= reported segment as .done using alphanumeric comparison on segment part (timeline-safe) - Archiver skips during recovery in shared mode, activates on promotion - Cascading replication: each standby coordinates with immediate upstream - Startup check rejects archive_mode=on during recovery This "push" design (primary sends status) is more efficient than "pull" (standby queries per-segment), avoiding directory scans and stat() calls. Based on Heikki Linnakangas's 2014 design and Greenplum's production implementation, modernized for PostgreSQL 19. Includes TAP tests covering basic synchronization, promotion, cascading replication, and multiple standbys scenarios. --- doc/src/sgml/config.sgml | 36 ++- doc/src/sgml/high-availability.sgml | 72 ++++-- src/backend/access/transam/xlog.c | 1 + src/backend/postmaster/pgarch.c | 17 +- src/backend/replication/walreceiver.c | 146 +++++++++++- src/backend/replication/walsender.c | 93 ++++++++ src/include/access/xlog.h | 1 + src/include/libpq/protocol.h | 22 ++ src/test/recovery/t/050_archive_shared.pl | 270 ++++++++++++++++++++++ 9 files changed, 620 insertions(+), 38 deletions(-) create mode 100644 src/test/recovery/t/050_archive_shared.pl diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 880e83035dc..e8709df3c86 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3747,14 +3747,36 @@ include_dir 'conf.d' are sent to archive storage by setting or . In addition to off, - to disable, there are two modes: on, and - always. During normal operation, there is no - difference between the two modes, but when set to always - the WAL archiver is enabled also during archive recovery or standby - mode. In always mode, all files restored from the archive - or streamed with streaming replication will be archived (again). See - for details. + to disable, there are three modes: on, shared, + and always. During normal operation as a primary, there is no + difference between the three modes, but they differ during archive recovery or + standby mode: + + + + on: Archives WAL only when running as a primary. + + + + + shared: Coordinates archiving between primary and standby. + The standby defers WAL archival and deletion until the primary confirms + archival via streaming replication. This prevents WAL history loss during + standby promotion in high availability setups. Upon promotion, the standby + automatically starts archiving any remaining unarchived WAL. This mode works + with cascading replication, where each standby coordinates with its immediate + upstream server. See for details. + + + + + always: Archives all WAL independently, even during recovery. + All files restored from the archive or streamed with streaming physical + replication will be archived (again), regardless of their source. + + + archive_mode is a separate setting from archive_command and diff --git a/doc/src/sgml/high-availability.sgml b/doc/src/sgml/high-availability.sgml index acf3ac0601d..fc81deb5428 100644 --- a/doc/src/sgml/high-availability.sgml +++ b/doc/src/sgml/high-availability.sgml @@ -1385,35 +1385,61 @@ synchronous_standby_names = 'ANY 2 (s1, s2, s3)' - When continuous WAL archiving is used in a standby, there are two - different scenarios: the WAL archive can be shared between the primary - and the standby, or the standby can have its own WAL archive. When - the standby has its own WAL archive, set archive_mode + When continuous WAL archiving is used in a standby, there are three + different scenarios: the standby can have its own independent WAL archive, + the WAL archive can be shared between the primary and standby, or archiving + can be coordinated between them. + + + + For an independent archive, set archive_mode to always, and the standby will call the archive command for every WAL segment it receives, whether it's by restoring - from the archive or by streaming replication. The shared archive can - be handled similarly, but the archive_command or archive_library must - test if the file being archived exists already, and if the existing file - has identical contents. This requires more care in the - archive_command or archive_library, as it must - be careful to not overwrite an existing file with different contents, - but return success if the exactly same file is archived twice. And - all that must be done free of race conditions, if two servers attempt - to archive the same file at the same time. + from the archive or by streaming replication. + + + + For a shared archive where both primary and standby can write, use + always mode as well, but the archive_command + or archive_library must test if the file being archived + exists already, and if the existing file has identical contents. This requires + more care in the archive_command or archive_library, + as it must be careful to not overwrite an existing file with different contents, + but return success if the exactly same file is archived twice. And all that must + be done free of race conditions, if two servers attempt to archive the same file + at the same time. + + + + For coordinated archiving in high availability setups, use + archive_mode=shared. In this mode, only + the primary archives WAL segments. The standby creates .ready + files for received segments but defers actual archiving. The primary periodically + sends archival status updates to the standby via streaming replication, informing + it which segments have been archived. The standby then marks these as archived + and allows them to be recycled. Upon promotion, the standby automatically starts + archiving any remaining WAL segments that weren't confirmed as archived by the + former primary. This prevents WAL history loss during failover while avoiding + the complexity of coordinating concurrent archiving. This mode works with cascading + replication, where each standby coordinates with its immediate upstream server. If archive_mode is set to on, the - archiver is not enabled during recovery or standby mode. If the standby - server is promoted, it will start archiving after the promotion, but - will not archive any WAL or timeline history files that - it did not generate itself. To get a complete - series of WAL files in the archive, you must ensure that all WAL is - archived, before it reaches the standby. This is inherently true with - file-based log shipping, as the standby can only restore files that - are found in the archive, but not if streaming replication is enabled. - When a server is not in recovery mode, there is no difference between - on and always modes. + archiver is not enabled during recovery or standby mode, and this setting + cannot be used on a standby. If a standby with archive_mode + set to on is promoted, it will start archiving after the + promotion, but will not archive any WAL or timeline history files that it did + not generate itself. To get a complete series of WAL files in the archive, you + must ensure that all WAL is archived before it reaches the standby. This is + inherently true with file-based log shipping, as the standby can only restore + files that are found in the archive, but not if streaming replication is enabled. + + + + When a server is not in recovery mode, on, + shared, and always modes all behave + identically, archiving completed WAL segments. diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 745276908d2..7fffe99da39 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -193,6 +193,7 @@ const struct config_enum_entry archive_mode_options[] = { {"always", ARCHIVE_MODE_ALWAYS, false}, {"on", ARCHIVE_MODE_ON, false}, {"off", ARCHIVE_MODE_OFF, false}, + {"shared", ARCHIVE_MODE_SHARED, false}, {"true", ARCHIVE_MODE_ON, true}, {"false", ARCHIVE_MODE_OFF, true}, {"yes", ARCHIVE_MODE_ON, true}, diff --git a/src/backend/postmaster/pgarch.c b/src/backend/postmaster/pgarch.c index 02f91431f5f..331b2b48713 100644 --- a/src/backend/postmaster/pgarch.c +++ b/src/backend/postmaster/pgarch.c @@ -381,6 +381,15 @@ pgarch_ArchiverCopyLoop(void) { char xlog[MAX_XFN_CHARS + 1]; + /* + * In shared archive mode during recovery, the archiver doesn't archive + * files. The primary is responsible for archiving, and the walreceiver + * marks files as .done when the primary confirms archival. After + * promotion, the archiver starts working normally. + */ + if (XLogArchiveMode == ARCHIVE_MODE_SHARED && RecoveryInProgress()) + return; + /* force directory scan in the first call to pgarch_readyXlog() */ arch_files->arch_files_size = 0; @@ -471,10 +480,10 @@ pgarch_ArchiverCopyLoop(void) continue; } - if (pgarch_archiveXlog(xlog)) - { - /* successful */ - pgarch_archiveDone(xlog); + if (pgarch_archiveXlog(xlog)) + { + /* successful */ + pgarch_archiveDone(xlog); /* * Tell the cumulative stats system about the WAL file that we diff --git a/src/backend/replication/walreceiver.c b/src/backend/replication/walreceiver.c index acda5f68d9a..238ed6bf9b7 100644 --- a/src/backend/replication/walreceiver.c +++ b/src/backend/replication/walreceiver.c @@ -130,6 +130,11 @@ static TimestampTz wakeup[NUM_WALRCV_WAKEUPS]; static StringInfoData reply_message; +/* Last archived WAL segment file reported by the primary */ +static char primary_last_archived[MAX_XFN_CHARS + 1]; +static TimeLineID primary_last_archived_tli = 0; +static XLogSegNo primary_last_archived_segno = 0; + /* Prototypes for private functions */ static void WalRcvFetchTimeLineHistoryFiles(TimeLineID first, TimeLineID last); static void WalRcvWaitForStartPosition(XLogRecPtr *startpoint, TimeLineID *startpointTLI); @@ -143,6 +148,7 @@ static void XLogWalRcvClose(XLogRecPtr recptr, TimeLineID tli); static void XLogWalRcvSendReply(bool force, bool requestReply); static void XLogWalRcvSendHSFeedback(bool immed); static void ProcessWalSndrMessage(XLogRecPtr walEnd, TimestampTz sendTime); +static void ProcessArchivalReport(void); static void WalRcvComputeNextWakeup(WalRcvWakeupReason reason, TimestampTz now); /* @@ -895,6 +901,30 @@ XLogWalRcvProcessMsg(unsigned char type, char *buf, Size len, TimeLineID tli) XLogWalRcvSendReply(true, false); break; } + case PqReplMsg_ArchiveStatusReport: + { + /* Check that the filename looks valid */ + if (len >= sizeof(primary_last_archived)) + ereport(ERROR, + (errcode(ERRCODE_PROTOCOL_VIOLATION), + errmsg_internal("invalid archival report message with length %d", + (int) len))); + + memcpy(primary_last_archived, buf, len); + primary_last_archived[len] = '\0'; + + /* Verify it contains only valid characters */ + if (strspn(buf, VALID_XFN_CHARS) != len) + { + primary_last_archived[0] = '\0'; + ereport(ERROR, + (errcode(ERRCODE_PROTOCOL_VIOLATION), + errmsg_internal("unexpected character in primary's last archived filename"))); + } + + ProcessArchivalReport(); + break; + } default: ereport(ERROR, (errcode(ERRCODE_PROTOCOL_VIOLATION), @@ -1073,12 +1103,39 @@ XLogWalRcvClose(XLogRecPtr recptr, TimeLineID tli) /* * Create .done file forcibly to prevent the streamed segment from being - * archived later. + * archived later, unless archive_mode is 'always' or 'shared'. + * + * In 'always' mode, the standby archives independently. + * + * In 'shared' mode, we optimize by checking if this segment is already + * covered by the last archival report from the primary. If so, create + * .done directly. Otherwise, create .ready and wait for the next report. */ - if (XLogArchiveMode != ARCHIVE_MODE_ALWAYS) - XLogArchiveForceDone(xlogfname); - else + if (XLogArchiveMode == ARCHIVE_MODE_ALWAYS) + { XLogArchiveNotify(xlogfname); + } + else if (XLogArchiveMode == ARCHIVE_MODE_SHARED) + { + /* + * In shared mode, check if this segment is already archived on primary. + * If we're on the same timeline and this segment is <= last archived, + * mark it .done immediately. Otherwise create .ready. + */ + if (primary_last_archived_tli == recvFileTLI && + recvSegNo <= primary_last_archived_segno) + { + XLogArchiveForceDone(xlogfname); + } + else + { + XLogArchiveNotify(xlogfname); + } + } + else + { + XLogArchiveForceDone(xlogfname); + } recvFile = -1; } @@ -1255,6 +1312,87 @@ XLogWalRcvSendHSFeedback(bool immed) primary_has_standby_xmin = false; } +/* + * Process archival report from primary. + * + * The primary sends us the last WAL segment it has archived. We scan the + * archive_status directory for .ready files and mark segments on the same + * timeline as .done if they're <= the reported segment. + */ +static void +ProcessArchivalReport(void) +{ + TimeLineID reported_tli; + XLogSegNo reported_segno; + DIR *status_dir; + struct dirent *status_de; + char status_path[MAXPGPATH]; + + elog(DEBUG2, "received archival report from primary: %s", + primary_last_archived); + + /* Parse the reported WAL filename */ + if (!IsXLogFileName(primary_last_archived)) + { + elog(DEBUG2, "invalid WAL filename in archival report: %s", + primary_last_archived); + return; + } + + XLogFromFileName(primary_last_archived, &reported_tli, &reported_segno, + wal_segment_size); + + /* Remember the last archived segment for XLogWalRcvClose() */ + primary_last_archived_tli = reported_tli; + primary_last_archived_segno = reported_segno; + + /* Scan archive_status directory for .ready files */ + snprintf(status_path, MAXPGPATH, XLOGDIR "/archive_status"); + status_dir = AllocateDir(status_path); + if (status_dir == NULL) + { + elog(DEBUG2, "could not open archive_status directory: %m"); + return; + } + + while ((status_de = ReadDir(status_dir, status_path)) != NULL) + { + char *ready_suffix; + char walfile[MAXPGPATH]; + TimeLineID file_tli; + XLogSegNo file_segno; + /* Look for .ready files only */ + ready_suffix = strstr(status_de->d_name, ".ready"); + if (ready_suffix == NULL || ready_suffix[6] != '\0') + continue; + + /* Extract WAL filename (remove .ready suffix) */ + strlcpy(walfile, status_de->d_name, ready_suffix - status_de->d_name + 1); + + /* Parse the WAL filename */ + if (!IsXLogFileName(walfile)) + continue; + + XLogFromFileName(walfile, &file_tli, &file_segno, wal_segment_size); + + /* + * Mark as .done if it's on the same timeline and not after the + * reported segment. We only process the reported timeline to avoid + * marking segments from parent or future timelines prematurely. + * XXX: Process possible TLI switches happened between status reports. + * For now, leave segments on previous TLIs to archive_command. + */ + if (file_tli == reported_tli && file_segno <= reported_segno) + { + XLogArchiveForceDone(walfile); + elog(DEBUG3, "marked WAL segment %s as archived (primary archived up to %s)", + walfile, primary_last_archived); + } + } + + FreeDir(status_dir); +} + /* * Update shared memory status upon receiving a message from primary. * diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index dceb696b86f..d4927e98ae0 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -192,6 +192,17 @@ static TimestampTz last_reply_timestamp = 0; /* Have we sent a heartbeat message asking for reply, since last reply? */ static bool waiting_for_ping_response = false; +/* + * Last archived WAL file. This is fetched from pgstat periodically and sent + * to the standby. last_archival_report_timestamp tracks when we last sent + * the report to avoid excessive pgstat access. + */ +static char last_archived_wal[MAX_XFN_CHARS + 1]; +static TimestampTz last_archival_report_timestamp = 0; + +/* Interval for sending archival reports (10 seconds) */ +#define ARCHIVAL_REPORT_INTERVAL 10000 + /* * While streaming WAL in Copy mode, streamingDoneSending is set to true * after we have sent CopyDone. We should not send any more CopyData messages @@ -292,6 +303,7 @@ static void StartLogicalReplication(StartReplicationCmd *cmd); static void ProcessStandbyMessage(void); static void ProcessStandbyReplyMessage(void); static void ProcessStandbyHSFeedbackMessage(void); +static void WalSndArchivalReport(void); static void ProcessRepliesIfAny(void); static void ProcessPendingWrites(void); static void WalSndKeepalive(bool requestReply, XLogRecPtr writePtr); @@ -2755,6 +2767,84 @@ ProcessStandbyHSFeedbackMessage(void) } } +/* + * Send archival status report to standby. + * + * This is called periodically during physical replication to inform the + * standby about the last WAL segment archived by the primary. The standby + * can then mark segments up to that point as .done, allowing them to be + * recycled. This prevents WAL loss during standby promotion. + */ +static void +WalSndArchivalReport(void) +{ + PgStat_ArchiverStats *archiver_stats; + TimestampTz now; + char *last_archived; + + /* Only send reports when archive_mode=shared */ + if (XLogArchiveMode != ARCHIVE_MODE_SHARED) + return; + + /* Only send reports during physical streaming replication, not during backup */ + if (MyWalSnd->kind != REPLICATION_KIND_PHYSICAL) + return; + if (MyWalSnd->state != WALSNDSTATE_CATCHUP && + MyWalSnd->state != WALSNDSTATE_STREAMING) + return; + + /* + * Don't send to temporary replication slots (used by pg_basebackup). + * Connections without slots (regular standbys) are OK. + */ + if (MyReplicationSlot != NULL && + MyReplicationSlot->data.persistency == RS_TEMPORARY) + return; + + now = GetCurrentTimestamp(); + + /* + * Send report at most once per ARCHIVAL_REPORT_INTERVAL (10 seconds). + * This avoids excessive pgstat access. + */ + if (now < TimestampTzPlusMilliseconds(last_archival_report_timestamp, + ARCHIVAL_REPORT_INTERVAL)) + return; + last_archival_report_timestamp = now; + /* + * Get archiver statistics. We use non-blocking access to avoid delaying + * replication if stats collector is slow. If stats are unavailable or + * stale, we'll just try again at the next interval. + */ + archiver_stats = pgstat_fetch_stat_archiver(); + if (archiver_stats == NULL) + return; + + last_archived = archiver_stats->last_archived_wal; + /* + * Only send a report if the last archived WAL has changed. This is both + * an optimization and ensures we don't send empty reports on startup. + */ + if (strcmp(last_archived, last_archived_wal) == 0) + return; + + /* Only send reports for WAL segments, not backup history files or other archived files */ + if (!IsXLogFileName(last_archived)) + return; + + elog(DEBUG2, "sending archival report: %s", last_archived); + + /* Remember what we sent */ + strlcpy(last_archived_wal, last_archived, sizeof(last_archived_wal)); + + /* Construct the message... */ + resetStringInfo(&output_message); + pq_sendbyte(&output_message, PqReplMsg_ArchiveStatusReport); + pq_sendbytes(&output_message, last_archived, strlen(last_archived)); + /* ... and send it wrapped in CopyData */ + pq_putmessage_noblock(PqMsg_CopyData, output_message.data, output_message.len); +} + /* * Compute how long send/receive loops should sleep. * @@ -4176,6 +4266,9 @@ WalSndKeepaliveIfNecessary(void) if (pq_flush_if_writable() != 0) WalSndShutdown(); } + + /* Send archival status report if needed */ + WalSndArchivalReport(); } /* diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h index 30f5aa6f151..c802d173694 100644 --- a/src/include/access/xlog.h +++ b/src/include/access/xlog.h @@ -63,6 +63,7 @@ typedef enum ArchiveMode ARCHIVE_MODE_OFF = 0, /* disabled */ ARCHIVE_MODE_ON, /* enabled while server is running normally */ ARCHIVE_MODE_ALWAYS, /* enabled always (even during recovery) */ + ARCHIVE_MODE_SHARED, /* shared archive between primary and standby */ } ArchiveMode; extern PGDLLIMPORT int XLogArchiveMode; diff --git a/src/include/libpq/protocol.h b/src/include/libpq/protocol.h index b71add1ec15..20b0871e90d 100644 --- a/src/include/libpq/protocol.h +++ b/src/include/libpq/protocol.h @@ -69,6 +69,28 @@ #define PqMsg_Progress 'P' +/* Replication codes sent by the primary (wrapped in CopyData messages). */ + +#define PqReplMsg_ArchiveStatusReport 'a' +#define PqReplMsg_Keepalive 'k' +#define PqReplMsg_PrimaryStatusUpdate 's' +#define PqReplMsg_WALData 'w' + + +/* Replication codes sent by the standby (wrapped in CopyData messages). */ + +#define PqReplMsg_HotStandbyFeedback 'h' +#define PqReplMsg_PrimaryStatusRequest 'p' +#define PqReplMsg_StandbyStatusUpdate 'r' + + +/* Codes used for backups via COPY OUT (wrapped in CopyData messages). */ + +#define PqBackupMsg_Manifest 'm' +#define PqBackupMsg_NewArchive 'n' +#define PqBackupMsg_ProgressReport 'p' + + /* These are the authentication request codes sent by the backend. */ #define AUTH_REQ_OK 0 /* User is authenticated */ diff --git a/src/test/recovery/t/050_archive_shared.pl b/src/test/recovery/t/050_archive_shared.pl new file mode 100644 index 00000000000..397b71ad79d --- /dev/null +++ b/src/test/recovery/t/050_archive_shared.pl @@ -0,0 +1,270 @@ +# Copyright (c) 2025, PostgreSQL Global Development Group + +# Test archive_mode=shared for coordinated WAL archiving between primary and standby +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; +use File::Path qw(rmtree); + +# Initialize primary node with archiving +my $archive_dir = PostgreSQL::Test::Utils::tempdir(); +my $primary = PostgreSQL::Test::Cluster->new('primary'); +$primary->init(has_archiving => 1, allows_streaming => 1); +$primary->append_conf('postgresql.conf', " +archive_mode = shared +archive_command = 'cp %p \"$archive_dir\"/%f' +wal_keep_size = 128MB +"); +$primary->start; + +# Create a test table and generate some WAL +$primary->safe_psql('postgres', 'CREATE TABLE test_table (id int, data text);'); +$primary->safe_psql('postgres', "INSERT INTO test_table SELECT i, 'data' || i FROM generate_series(1, 500) i;"); +$primary->safe_psql('postgres', 'SELECT pg_switch_wal();'); +$primary->safe_psql('postgres', "INSERT INTO test_table SELECT i, 'data' || i FROM generate_series(501, 1000) i;"); +$primary->safe_psql('postgres', 'SELECT pg_switch_wal();'); + +# Wait for archiver to archive segments +$primary->poll_query_until('postgres', + "SELECT archived_count > 0 FROM pg_stat_archiver") + or die "Timed out waiting for archiver to start"; + +my $archived_count = () = glob("$archive_dir/*"); +ok($archived_count > 0, "primary has archived WAL files to shared archive"); +note("Primary archived $archived_count files"); + +# Take backup for standby +my $backup_name = 'standby_backup'; +$primary->backup($backup_name); + +# Exclude possible race condition when backup WAL is last archived +$primary->safe_psql('postgres', "INSERT INTO test_table SELECT i, 'data' || i FROM generate_series(501, 1000) i;"); +$primary->safe_psql('postgres', 'SELECT pg_switch_wal();'); + +# Set up standby with archive_mode=shared +my $standby = PostgreSQL::Test::Cluster->new('standby'); +$standby->init_from_backup($primary, $backup_name, has_streaming => 1); +$standby->append_conf('postgresql.conf', " +archive_mode = shared +archive_command = 'cp %p \"$archive_dir\"/%f' +wal_receiver_status_interval = 1s +"); +$standby->start; + +# Wait for standby to catch up +$primary->wait_for_catchup($standby); + +# Generate more WAL on primary (these are new segments not yet archived) +$primary->safe_psql('postgres', "INSERT INTO test_table SELECT i, 'data' || i FROM generate_series(1001, 1500) i;"); +$primary->safe_psql('postgres', 'SELECT pg_switch_wal();'); +$primary->safe_psql('postgres', "INSERT INTO test_table SELECT i, 'data' || i FROM generate_series(1501, 2000) i;"); +$primary->safe_psql('postgres', 'SELECT pg_switch_wal();'); + +# Wait for standby to receive the new WAL +$primary->wait_for_catchup($standby); + +# Check that standby has .ready or .done files for the newly received segments. +# Normally they should be .ready (not yet archived by primary), but in rare cases +# the archiver could be very fast and an archive report sent immediately, creating +# .done files instead. Both are correct behavior - the key is that files exist. +my $standby_archive_status = $standby->data_dir . '/pg_wal/archive_status'; +my $status_count = 0; +if (opendir(my $dh, $standby_archive_status)) +{ + my @files = grep { /\.(ready|done)$/ } readdir($dh); + $status_count = scalar(@files); + my $ready_count = scalar(grep { /\.ready$/ } @files); + my $done_count = scalar(grep { /\.done$/ } @files); + note("Standby has $ready_count .ready files and $done_count .done files"); + closedir($dh); +} +cmp_ok($status_count, '>', 0, "standby creates archive status files for received WAL"); + +# Generate more WAL and wait for archiving on primary +my $initial_archived = $primary->safe_psql('postgres', 'SELECT archived_count FROM pg_stat_archiver'); +$primary->safe_psql('postgres', "INSERT INTO test_table SELECT i, 'more-data' || i FROM generate_series(2001, 2500) i;"); +$primary->safe_psql('postgres', 'SELECT pg_switch_wal();'); +$primary->safe_psql('postgres', "INSERT INTO test_table SELECT i, 'more-data2' || i FROM generate_series(2501, 3000) i;"); +$primary->safe_psql('postgres', 'SELECT pg_switch_wal();'); + +# Wait for primary to archive the new segments +$primary->poll_query_until('postgres', + "SELECT archived_count > $initial_archived FROM pg_stat_archiver") + or die "Timed out waiting for primary to archive new segments"; + +# Wait for standby to catch up (archive status is sent during replication) +$primary->wait_for_catchup($standby); + +# Wait for primary to send archival status updates and standby to process them +# The standby should mark segments as .done after receiving archive status from primary +my $done_count = 0; +for (my $i = 0; $i < $PostgreSQL::Test::Utils::timeout_default; $i++) +{ + $done_count = 0; + if (opendir(my $dh, $standby_archive_status)) + { + $done_count = scalar(grep { /\.done$/ } readdir($dh)); + closedir($dh); + } + last if $done_count > 0; + sleep(1); +} +ok($done_count > 0, "standby marked segments as .done after primary's archival report"); +note("Standby has $done_count .done files"); + +############################################################################### +# Test 2: Standby promotion - verify archiver activates +############################################################################### + +# Before promotion, verify archiver is not running on standby (shared mode during recovery) +# In shared mode, the standby's archiver should not be archiving during recovery +my $archived_before = $standby->safe_psql('postgres', + "SELECT archived_count FROM pg_stat_archiver"); +is($archived_before, '0', + "archiver not active on standby before promotion (archived_count=0)"); + +# Verify standby is still in recovery before promoting +my $in_recovery = $standby->safe_psql('postgres', "SELECT pg_is_in_recovery();"); +is($in_recovery, 't', "standby is in recovery before promotion"); + +# Promote the standby +$standby->promote; +$standby->poll_query_until('postgres', "SELECT NOT pg_is_in_recovery();"); + +# Generate WAL on new primary (former standby) +$standby->safe_psql('postgres', "INSERT INTO test_table SELECT i, 'post-promotion' || i FROM generate_series(2001, 2500) i;"); +$standby->safe_psql('postgres', 'SELECT pg_switch_wal();'); + +# Wait for archiver to activate and archive the new WAL +# Check pg_stat_archiver to verify archiving is happening +$standby->poll_query_until('postgres', + "SELECT archived_count > 0 FROM pg_stat_archiver") + or die "Timed out waiting for promoted standby to start archiving"; +pass("promoted standby started archiving"); + +# Verify data integrity +my $count = $standby->safe_psql('postgres', 'SELECT COUNT(*) FROM test_table;'); +ok($count >= 2500, "promoted standby has all data (got $count rows)"); + +############################################################################### +# Test 3: Cascading replication +############################################################################### + +# Take a backup from the promoted standby (now the new primary) +my $promoted_backup = 'promoted_backup'; +$standby->backup($promoted_backup); + +# Set up second-level standby (cascading from first standby, now promoted) +my $standby2 = PostgreSQL::Test::Cluster->new('standby2'); +$standby2->init_from_backup($standby, $promoted_backup, has_streaming => 1); +$standby2->append_conf('postgresql.conf', " +archive_mode = shared +archive_command = 'cp %p \"$archive_dir\"/%f' +wal_receiver_status_interval = 1s +"); +$standby2->start; + +# Generate WAL on promoted standby (now primary for standby2) +my $cascading_archived_before = $standby->safe_psql('postgres', 'SELECT archived_count FROM pg_stat_archiver'); +$standby->safe_psql('postgres', "INSERT INTO test_table SELECT i, 'cascading' || i FROM generate_series(2501, 3000) i;"); +$standby->safe_psql('postgres', 'SELECT pg_switch_wal();'); + +# Wait for the promoted standby (acting as primary) to archive the new segment +$standby->poll_query_until('postgres', + "SELECT archived_count > $cascading_archived_before FROM pg_stat_archiver") + or die "Timed out waiting for primary to archive segment in cascading test"; + +# Wait for cascading standby to catch up +$standby->wait_for_catchup($standby2); + +# Wait for cascading standby to receive archive status and mark segments as .done +my $standby2_archive_status = $standby2->data_dir . '/pg_wal/archive_status'; +my $standby2_done_count = 0; +for (my $i = 0; $i < $PostgreSQL::Test::Utils::timeout_default; $i++) +{ + $standby2_done_count = 0; + if (opendir(my $dh, $standby2_archive_status)) + { + $standby2_done_count = scalar(grep { /\.done$/ } readdir($dh)); + closedir($dh); + } + last if $standby2_done_count > 0; + sleep(1); +} +ok($standby2_done_count > 0, "cascading standby marks segments as .done"); +note("Cascading standby has $standby2_done_count .done files"); + +# Verify cascading standby has all data +my $standby2_count = $standby2->safe_psql('postgres', 'SELECT COUNT(*) FROM test_table;'); +ok($standby2_count >= 3000, "cascading standby has all data (got $standby2_count rows)"); + +############################################################################### +# Test 4: Multiple standbys from same primary +############################################################################### + +# Create third standby from promoted standby (current primary) +my $standby3 = PostgreSQL::Test::Cluster->new('standby3'); +my $backup2 = 'multi_standby_backup'; +$standby->backup($backup2); +$standby3->init_from_backup($standby, $backup2, has_streaming => 1); +$standby3->append_conf('postgresql.conf', " +archive_mode = shared +archive_command = 'cp %p \"$archive_dir\"/%f' +wal_receiver_status_interval = 1s +"); +$standby3->start; + +# Generate WAL and ensure both standbys receive it +my $standby_archived_before = $standby->safe_psql('postgres', 'SELECT archived_count FROM pg_stat_archiver'); +$standby->safe_psql('postgres', "INSERT INTO test_table SELECT i, 'multi' || i FROM generate_series(3001, 3500) i;"); +$standby->safe_psql('postgres', 'SELECT pg_switch_wal();'); + +# Wait for the promoted standby (acting as primary) to archive the new segment +$standby->poll_query_until('postgres', + "SELECT archived_count > $standby_archived_before FROM pg_stat_archiver") + or die "Timed out waiting for primary to archive segment in multi-standby test"; + +$standby->wait_for_catchup($standby2); +$standby->wait_for_catchup($standby3); + +# Verify both standbys eventually mark segments as .done +my $standby3_archive_status = $standby3->data_dir . '/pg_wal/archive_status'; + +for (my $i = 0; $i < $PostgreSQL::Test::Utils::timeout_default; $i++) +{ + $standby2_done_count = 0; + if (opendir(my $dh, $standby2_archive_status)) + { + $standby2_done_count = scalar(grep { /\.done$/ } readdir($dh)); + closedir($dh); + } + last if $standby2_done_count > 0; + sleep(1); +} + +my $standby3_done_count = 0; +for (my $i = 0; $i < $PostgreSQL::Test::Utils::timeout_default; $i++) +{ + $standby3_done_count = 0; + if (opendir(my $dh, $standby3_archive_status)) + { + $standby3_done_count = scalar(grep { /\.done$/ } readdir($dh)); + closedir($dh); + } + last if $standby3_done_count > 0; + sleep(1); +} + +ok($standby2_done_count > 0, "standby2 marks segments as .done"); +ok($standby3_done_count > 0, "standby3 marks segments as .done"); +note("standby2 has $standby2_done_count .done files, standby3 has $standby3_done_count .done files"); + +# Verify both standbys have all data +$standby2_count = $standby2->safe_psql('postgres', 'SELECT COUNT(*) FROM test_table;'); +my $standby3_count = $standby3->safe_psql('postgres', 'SELECT COUNT(*) FROM test_table;'); +ok($standby2_count >= 3500, "standby2 has all data (got $standby2_count rows)"); +ok($standby3_count >= 3500, "standby3 has all data (got $standby3_count rows)"); + +done_testing(); From 7feecff45067d91153582c87ca677e40fbcc4864 Mon Sep 17 00:00:00 2001 From: Andrey Borodin Date: Tue, 10 Feb 2026 16:45:10 +0500 Subject: [PATCH 50/54] Mark ancestor timeline WAL segments as archived When standby receives archive status report, check if .ready files belong to ancestor timelines before the switch point and mark them as .done if already archived by primary. --- src/backend/replication/walreceiver.c | 55 ++++++++++++++++++++++++--- 1 file changed, 50 insertions(+), 5 deletions(-) diff --git a/src/backend/replication/walreceiver.c b/src/backend/replication/walreceiver.c index 238ed6bf9b7..9d135821ae6 100644 --- a/src/backend/replication/walreceiver.c +++ b/src/backend/replication/walreceiver.c @@ -1121,6 +1121,11 @@ XLogWalRcvClose(XLogRecPtr recptr, TimeLineID tli) * In shared mode, check if this segment is already archived on primary. * If we're on the same timeline and this segment is <= last archived, * mark it .done immediately. Otherwise create .ready. + * + * We don't check ancestor timeline cases here to avoid reading timeline + * history files on every segment close. ProcessArchivalReport() will + * handle marking ancestor timeline segments as .done when it scans + * the archive_status directory. */ if (primary_last_archived_tli == recvFileTLI && recvSegNo <= primary_last_archived_segno) @@ -1327,6 +1332,7 @@ ProcessArchivalReport(void) DIR *status_dir; struct dirent *status_de; char status_path[MAXPGPATH]; + List *tli_history = NIL; elog(DEBUG2, "received archival report from primary: %s", primary_last_archived); @@ -1376,18 +1382,57 @@ ProcessArchivalReport(void) XLogFromFileName(walfile, &file_tli, &file_segno, wal_segment_size); /* - * Mark as .done if it's on the same timeline and not after the - * reported segment. We only process the reported timeline to avoid - * marking segments from parent or future timelines prematurely. - * XXX: Process possible TLI switches happened between status reports. - * For now, leave segments on previous TLIs to archive_command. + * Mark as .done if: + * 1. Same timeline and segment <= reported segment, OR + * 2. Ancestor timeline and segment is before the timeline switch point + * + * For ancestor timelines: if primary archived segment X on timeline T, + * then all segments on ancestor timelines before the switch to T must + * have been archived (they're required to reach timeline T). */ if (file_tli == reported_tli && file_segno <= reported_segno) { + /* Same timeline, segment already archived */ XLogArchiveForceDone(walfile); elog(DEBUG3, "marked WAL segment %s as archived (primary archived up to %s)", walfile, primary_last_archived); } + else if (file_tli != reported_tli) + { + /* + * Different timeline - check if it's an ancestor and if this + * segment is before the timeline switch point. Only read timeline + * history if we haven't already (lazy loading). + * + * Note: Timelines form a tree structure, not a linear sequence, + * so we can't use < or > to compare them. + */ + if (tli_history == NIL) + tli_history = readTimeLineHistory(reported_tli); + + if (tliInHistory(file_tli, tli_history)) + { + XLogRecPtr switchpoint; + XLogSegNo switchpoint_segno; + + /* Get the point where we switched away from this timeline */ + switchpoint = tliSwitchPoint(file_tli, tli_history, NULL); + + /* + * If the segment is at or before the switch point, it must have + * been archived (it's required to reach the reported timeline). + * The segment containing the switch point belongs to the old + * timeline up to the switch point and should be archived. + */ + XLByteToSeg(switchpoint, switchpoint_segno, wal_segment_size); + if (file_segno <= switchpoint_segno) + { + XLogArchiveForceDone(walfile); + elog(DEBUG3, "marked ancestor timeline segment %s as archived (before switch to timeline %u)", + walfile, reported_tli); + } + } + } } FreeDir(status_dir); From 48c845951009451e213b612613e4cca5491705f7 Mon Sep 17 00:00:00 2001 From: Andrey Borodin Date: Wed, 11 Feb 2026 18:17:25 +0500 Subject: [PATCH 51/54] Optimize ProcessArchivalReport to avoid directory scans When archive status reports arrive sequentially on the same timeline, directly generate expected WAL filenames and mark them as archived instead of scanning the entire archive_status directory. This optimization reduces overhead in the common case where the primary continuously archives segments. Directory scan is still used when: - Timeline changes (to handle ancestor timelines) - First report received - Non-sequential reports XLogArchiveForceDone() handles all cases internally (checking if .done exists, if .ready exists, or creating .done if neither exists), so no pre-check is needed. --- src/backend/replication/walreceiver.c | 196 +++++++++++++++++--------- 1 file changed, 132 insertions(+), 64 deletions(-) diff --git a/src/backend/replication/walreceiver.c b/src/backend/replication/walreceiver.c index 9d135821ae6..a049ddf92c2 100644 --- a/src/backend/replication/walreceiver.c +++ b/src/backend/replication/walreceiver.c @@ -135,6 +135,14 @@ static char primary_last_archived[MAX_XFN_CHARS + 1]; static TimeLineID primary_last_archived_tli = 0; static XLogSegNo primary_last_archived_segno = 0; +/* + * Last segment we successfully marked as .done. Used to optimize + * ProcessArchivalReport() by generating expected filenames instead + * of scanning the archive_status directory. + */ +static TimeLineID last_processed_tli = 0; +static XLogSegNo last_processed_segno = 0; + /* Prototypes for private functions */ static void WalRcvFetchTimeLineHistoryFiles(TimeLineID first, TimeLineID last); static void WalRcvWaitForStartPosition(XLogRecPtr *startpoint, TimeLineID *startpointTLI); @@ -1329,10 +1337,9 @@ ProcessArchivalReport(void) { TimeLineID reported_tli; XLogSegNo reported_segno; - DIR *status_dir; - struct dirent *status_de; char status_path[MAXPGPATH]; - List *tli_history = NIL; + bool use_direct_check = false; + XLogSegNo start_segno; elog(DEBUG2, "received archival report from primary: %s", primary_last_archived); @@ -1352,90 +1359,151 @@ ProcessArchivalReport(void) primary_last_archived_tli = reported_tli; primary_last_archived_segno = reported_segno; - /* Scan archive_status directory for .ready files */ - snprintf(status_path, MAXPGPATH, XLOGDIR "/archive_status"); - status_dir = AllocateDir(status_path); - if (status_dir == NULL) + /* + * Optimization: If the new report is on the same timeline as the last + * processed segment and moves forward, we can directly check for .ready + * files for segments between last_processed_segno and reported_segno + * instead of scanning the entire archive_status directory. + * + * Fall back to directory scan if: + * - Timeline changed (need to handle ancestor timelines) + * - This is the first report (last_processed_tli == 0) + * - Reported segment is not ahead (nothing new to process) + */ + if (last_processed_tli == reported_tli && + last_processed_tli != 0 && + reported_segno > last_processed_segno) { - elog(DEBUG2, "could not open archive_status directory: %m"); - return; + use_direct_check = true; + start_segno = last_processed_segno + 1; } - while ((status_de = ReadDir(status_dir, status_path)) != NULL) + if (use_direct_check) { - char *ready_suffix; - char walfile[MAXPGPATH]; - TimeLineID file_tli; - XLogSegNo file_segno; - /* Look for .ready files only */ - ready_suffix = strstr(status_de->d_name, ".ready"); - if (ready_suffix == NULL || ready_suffix[6] != '\0') - continue; - - /* Extract WAL filename (remove .ready suffix) */ - strlcpy(walfile, status_de->d_name, ready_suffix - status_de->d_name + 1); - - /* Parse the WAL filename */ - if (!IsXLogFileName(walfile)) - continue; - - XLogFromFileName(walfile, &file_tli, &file_segno, wal_segment_size); - /* - * Mark as .done if: - * 1. Same timeline and segment <= reported segment, OR - * 2. Ancestor timeline and segment is before the timeline switch point - * - * For ancestor timelines: if primary archived segment X on timeline T, - * then all segments on ancestor timelines before the switch to T must - * have been archived (they're required to reach timeline T). + * Direct check: generate filenames for expected segments. + * XLogArchiveForceDone() will handle the case where .ready doesn't + * exist or .done already exists, so no need to stat() first. */ - if (file_tli == reported_tli && file_segno <= reported_segno) + XLogSegNo segno; + + for (segno = start_segno; segno <= reported_segno; segno++) { - /* Same timeline, segment already archived */ + char walfile[MAXFNAMELEN]; + + /* Generate WAL filename and mark as archived */ + XLogFileName(walfile, reported_tli, segno, wal_segment_size); XLogArchiveForceDone(walfile); elog(DEBUG3, "marked WAL segment %s as archived (primary archived up to %s)", walfile, primary_last_archived); + + /* Track the last segment we processed */ + last_processed_tli = reported_tli; + last_processed_segno = segno; + } + } + else + { + /* + * Directory scan: needed when timeline changed or first report. + * This handles both same-timeline and ancestor-timeline cases. + */ + DIR *status_dir; + struct dirent *status_de; + List *tli_history = NIL; + + snprintf(status_path, MAXPGPATH, XLOGDIR "/archive_status"); + status_dir = AllocateDir(status_path); + if (status_dir == NULL) + { + elog(DEBUG2, "could not open archive_status directory: %m"); + return; } - else if (file_tli != reported_tli) + + while ((status_de = ReadDir(status_dir, status_path)) != NULL) { + char *ready_suffix; + char walfile[MAXPGPATH]; + TimeLineID file_tli; + XLogSegNo file_segno; + + /* Look for .ready files only */ + ready_suffix = strstr(status_de->d_name, ".ready"); + if (ready_suffix == NULL || ready_suffix[6] != '\0') + continue; + + /* Extract WAL filename (remove .ready suffix) */ + strlcpy(walfile, status_de->d_name, ready_suffix - status_de->d_name + 1); + + /* Parse the WAL filename */ + if (!IsXLogFileName(walfile)) + continue; + + XLogFromFileName(walfile, &file_tli, &file_segno, wal_segment_size); + /* - * Different timeline - check if it's an ancestor and if this - * segment is before the timeline switch point. Only read timeline - * history if we haven't already (lazy loading). + * Mark as .done if: + * 1. Same timeline and segment <= reported segment, OR + * 2. Ancestor timeline and segment is before the timeline switch point * - * Note: Timelines form a tree structure, not a linear sequence, - * so we can't use < or > to compare them. + * For ancestor timelines: if primary archived segment X on timeline T, + * then all segments on ancestor timelines before the switch to T must + * have been archived (they're required to reach timeline T). */ - if (tli_history == NIL) - tli_history = readTimeLineHistory(reported_tli); - - if (tliInHistory(file_tli, tli_history)) + if (file_tli == reported_tli && file_segno <= reported_segno) + { + /* Same timeline, segment already archived */ + XLogArchiveForceDone(walfile); + elog(DEBUG3, "marked WAL segment %s as archived (primary archived up to %s)", + walfile, primary_last_archived); + } + else if (file_tli != reported_tli) { - XLogRecPtr switchpoint; - XLogSegNo switchpoint_segno; - - /* Get the point where we switched away from this timeline */ - switchpoint = tliSwitchPoint(file_tli, tli_history, NULL); - /* - * If the segment is at or before the switch point, it must have - * been archived (it's required to reach the reported timeline). - * The segment containing the switch point belongs to the old - * timeline up to the switch point and should be archived. + * Different timeline - check if it's an ancestor and if this + * segment is before the timeline switch point. Only read timeline + * history if we haven't already (lazy loading). + * + * Note: Timelines form a tree structure, not a linear sequence, + * so we can't use < or > to compare them. */ - XLByteToSeg(switchpoint, switchpoint_segno, wal_segment_size); - if (file_segno <= switchpoint_segno) + if (tli_history == NIL) + tli_history = readTimeLineHistory(reported_tli); + + if (tliInHistory(file_tli, tli_history)) { - XLogArchiveForceDone(walfile); - elog(DEBUG3, "marked ancestor timeline segment %s as archived (before switch to timeline %u)", - walfile, reported_tli); + XLogRecPtr switchpoint; + XLogSegNo switchpoint_segno; + + /* Get the point where we switched away from this timeline */ + switchpoint = tliSwitchPoint(file_tli, tli_history, NULL); + + /* + * If the segment is at or before the switch point, it must have + * been archived (it's required to reach the reported timeline). + * The segment containing the switch point belongs to the old + * timeline up to the switch point and should be archived. + */ + XLByteToSeg(switchpoint, switchpoint_segno, wal_segment_size); + if (file_segno <= switchpoint_segno) + { + XLogArchiveForceDone(walfile); + elog(DEBUG3, "marked ancestor timeline segment %s as archived (before switch to timeline %u)", + walfile, reported_tli); + } } } } - } - FreeDir(status_dir); + FreeDir(status_dir); + + /* + * After a full directory scan following a timeline change, update + * our tracking to the newly reported position for future optimizations. + */ + last_processed_tli = reported_tli; + last_processed_segno = reported_segno; + } } /* From d95045b925b689f9a6515736bda114ebdf147bf4 Mon Sep 17 00:00:00 2001 From: Andrey Borodin Date: Thu, 12 Feb 2026 22:25:09 +0500 Subject: [PATCH 52/54] Fuse shared archive with ycmdb.shared_archive --- src/backend/access/transam/xlog.c | 1 + src/backend/postmaster/pgarch.c | 2 +- src/backend/replication/walreceiver.c | 2 +- src/backend/replication/walsender.c | 5 +++-- src/backend/utils/misc/guc_tables.c | 12 ++++++++++++ src/include/access/xlog.h | 9 +++++++++ 6 files changed, 27 insertions(+), 4 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 7fffe99da39..ebf3dfe468a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -118,6 +118,7 @@ int wal_keep_size_mb = 0; int XLOGbuffers = -1; int XLogArchiveTimeout = 0; int XLogArchiveMode = ARCHIVE_MODE_OFF; +bool ycmdb_shared_archive = false; /* makes archive_mode=on act as shared */ char *XLogArchiveCommand = NULL; bool EnableHotStandby = false; bool fullPageWrites = true; diff --git a/src/backend/postmaster/pgarch.c b/src/backend/postmaster/pgarch.c index 331b2b48713..42050c3314d 100644 --- a/src/backend/postmaster/pgarch.c +++ b/src/backend/postmaster/pgarch.c @@ -387,7 +387,7 @@ pgarch_ArchiverCopyLoop(void) * marks files as .done when the primary confirms archival. After * promotion, the archiver starts working normally. */ - if (XLogArchiveMode == ARCHIVE_MODE_SHARED && RecoveryInProgress()) + if (EffectiveArchiveModeIsShared() && RecoveryInProgress()) return; /* force directory scan in the first call to pgarch_readyXlog() */ diff --git a/src/backend/replication/walreceiver.c b/src/backend/replication/walreceiver.c index a049ddf92c2..d4900df98e0 100644 --- a/src/backend/replication/walreceiver.c +++ b/src/backend/replication/walreceiver.c @@ -1123,7 +1123,7 @@ XLogWalRcvClose(XLogRecPtr recptr, TimeLineID tli) { XLogArchiveNotify(xlogfname); } - else if (XLogArchiveMode == ARCHIVE_MODE_SHARED) + else if (EffectiveArchiveModeIsShared()) { /* * In shared mode, check if this segment is already archived on primary. diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index d4927e98ae0..7901c7ecc8c 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -83,6 +83,7 @@ #include "storage/ipc.h" #include "storage/pmsignal.h" #include "storage/proc.h" +#include "storage/procarray.h" #include "tcop/dest.h" #include "tcop/tcopprot.h" #include "utils/acl.h" @@ -2782,8 +2783,8 @@ WalSndArchivalReport(void) TimestampTz now; char *last_archived; - /* Only send reports when archive_mode=shared */ - if (XLogArchiveMode != ARCHIVE_MODE_SHARED) + /* Only send reports when shared archive is active */ + if (!EffectiveArchiveModeIsShared()) return; /* Only send reports during physical streaming replication, not during backup */ diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index c25ff52a585..9f7de099aab 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -31,6 +31,7 @@ #include "access/slru.h" #include "access/toast_compression.h" #include "access/twophase.h" +#include "access/xlog.h" #include "access/xlog_internal.h" #include "access/xlogprefetcher.h" #include "access/xlogrecovery.h" @@ -1190,6 +1191,17 @@ struct config_bool ConfigureNamesBool[] = NULL, NULL, NULL }, + { + {"ycmdb.shared_archive", PGC_POSTMASTER, WAL_ARCHIVING, + gettext_noop("Makes archive_mode=on behave as shared (for managed service compatibility)."), + gettext_noop("When true, archive_mode=on is treated as archive_mode=shared. Does not affect archive_mode=off or archive_mode=always. Used when control plane cannot configure archive_mode=shared directly."), + GUC_NOT_IN_SAMPLE + }, + &ycmdb_shared_archive, + false, + NULL, NULL, NULL + }, + { {"wal_init_zero", PGC_SUSET, WAL_SETTINGS, gettext_noop("Writes zeroes to new WAL files before first use."), diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h index c802d173694..0d7e55146ff 100644 --- a/src/include/access/xlog.h +++ b/src/include/access/xlog.h @@ -66,6 +66,15 @@ typedef enum ArchiveMode ARCHIVE_MODE_SHARED, /* shared archive between primary and standby */ } ArchiveMode; extern PGDLLIMPORT int XLogArchiveMode; +extern PGDLLIMPORT bool ycmdb_shared_archive; + +/* + * True when shared archive behavior is active: either archive_mode=shared + * or archive_mode=on with ycmdb.shared_archive=true (managed service). + */ +#define EffectiveArchiveModeIsShared() \ + (XLogArchiveMode == ARCHIVE_MODE_SHARED || \ + (XLogArchiveMode == ARCHIVE_MODE_ON && ycmdb_shared_archive)) /* WAL levels */ typedef enum WalLevel From 1ff9060ba5e091c7210677e34a3e769602a2a55a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lvaro=20Herrera?= Date: Thu, 1 Jan 2026 13:47:51 -0300 Subject: [PATCH 53/54] REASSIGN OWNED: ignore subscriptions in other databases XXX do we also need DROP OWNED to work this way? Probably not. Discussion: https://postgr.es/m/CALdSSPhjONb+EftRD=J1pqajkB+pjT0=tbMJs16C6q9+xT8NNg@mail.gmail.com --- src/backend/catalog/pg_shdepend.c | 10 ++++++++++ src/backend/utils/cache/lsyscache.c | 22 ++++++++++++++++++++++ src/include/utils/lsyscache.h | 1 + 3 files changed, 33 insertions(+) diff --git a/src/backend/catalog/pg_shdepend.c b/src/backend/catalog/pg_shdepend.c index 753afb88453..1be8d7d44c3 100644 --- a/src/backend/catalog/pg_shdepend.c +++ b/src/backend/catalog/pg_shdepend.c @@ -63,6 +63,7 @@ #include "utils/fmgroids.h" #include "utils/memutils.h" #include "utils/syscache.h" +#include "utils/lsyscache.h" typedef enum { @@ -1592,6 +1593,15 @@ shdepReassignOwned(List *roleids, Oid newrole) sdepForm->dbid != InvalidOid) continue; + /* + * Subscriptions are linked to specific databases, even though + * they are nominally shared objects. Skip those that aren't + * in this database. + */ + if (sdepForm->classid == SubscriptionRelationId && + get_subscription_database(sdepForm->objid) != MyDatabaseId) + continue; + /* * The various DDL routines called here tend to leak memory in * CurrentMemoryContext. That's not a problem when they're only diff --git a/src/backend/utils/cache/lsyscache.c b/src/backend/utils/cache/lsyscache.c index 48a280d089b..ea8f871e4eb 100644 --- a/src/backend/utils/cache/lsyscache.c +++ b/src/backend/utils/cache/lsyscache.c @@ -3714,3 +3714,25 @@ get_subscription_name(Oid subid, bool missing_ok) return subname; } + +/* + * Return the OID of the database the given subscription is in. + */ +Oid +get_subscription_database(Oid subid) +{ + HeapTuple tup; + Form_pg_subscription subform; + Oid subdbid; + + tup = SearchSysCache1(SUBSCRIPTIONOID, ObjectIdGetDatum(subid)); + if (!HeapTupleIsValid(tup)) + elog(ERROR, "cache lookup failed for subscription %u", subid); + + subform = (Form_pg_subscription) GETSTRUCT(tup); + subdbid = subform->subdbid; + + ReleaseSysCache(tup); + + return subdbid; +} diff --git a/src/include/utils/lsyscache.h b/src/include/utils/lsyscache.h index 20446f6f836..dbce680bbd3 100644 --- a/src/include/utils/lsyscache.h +++ b/src/include/utils/lsyscache.h @@ -205,6 +205,7 @@ extern Oid get_publication_oid(const char *pubname, bool missing_ok); extern char *get_publication_name(Oid pubid, bool missing_ok); extern Oid get_subscription_oid(const char *subname, bool missing_ok); extern char *get_subscription_name(Oid subid, bool missing_ok); +extern Oid get_subscription_database(Oid subid); #define type_is_array(typid) (get_element_type(typid) != InvalidOid) /* type_is_array_domain accepts both plain arrays and domains over arrays */ From 648ea5747ec69c8e4e3cc3826eaf2e7e3d016eb3 Mon Sep 17 00:00:00 2001 From: Andrey Borodin Date: Fri, 17 Apr 2026 23:06:08 +0500 Subject: [PATCH 54/54] Fix two bugs in archive_mode=shared on standby 1. Checkpoint on standby deletes WAL with .ready status. XLogArchiveCheckDone() treated archive_mode=shared like archive_mode=on during recovery, returning true unconditionally and allowing checkpoint to remove WAL segments that the primary had not yet archived. Fix: exclude shared mode from the early-return path, same as "always". 2. Walsender never sends archival status reports after archiving is restored. WalSndArchivalReport() calls pgstat_fetch_stat_archiver() whose result is cached per-session (PGSTAT_FETCH_CONSISTENCY_CACHE by default). The walsender has no transaction boundaries that would clear the cache, so last_archived_wal remained "" forever, and strcmp() suppressed all reports. Fix: call pgstat_clear_snapshot() before fetching archiver stats. Add TAP tests in 051_archive_shared_checkpoint.pl that reproduce both bugs, and extend 050_archive_shared.pl with checkpoint/restore scenarios. --- src/backend/access/transam/xlogarchive.c | 14 +- src/backend/replication/walsender.c | 9 +- src/test/recovery/t/050_archive_shared.pl | 138 ++++++++++++ .../t/051_archive_shared_checkpoint.pl | 211 ++++++++++++++++++ 4 files changed, 365 insertions(+), 7 deletions(-) create mode 100644 src/test/recovery/t/051_archive_shared_checkpoint.pl diff --git a/src/backend/access/transam/xlogarchive.c b/src/backend/access/transam/xlogarchive.c index 81999b48200..ebb498946d3 100644 --- a/src/backend/access/transam/xlogarchive.c +++ b/src/backend/access/transam/xlogarchive.c @@ -573,16 +573,22 @@ XLogArchiveCheckDone(const char *xlog) /* * During archive recovery, the file is deletable if archive_mode is not - * "always". + * "always" or "shared". + * + * In "shared" mode the standby does not archive independently; instead it + * waits for the primary to report successful archival, at which point the + * walreceiver converts the .ready file to .done. We must therefore fall + * through to the .done/.ready check below so that checkpoint cannot + * delete a segment whose .ready file has not yet become .done. */ - if (!XLogArchivingAlways() && + if (!XLogArchivingAlways() && !EffectiveArchiveModeIsShared() && GetRecoveryState() == RECOVERY_STATE_ARCHIVE) return true; /* * At this point of the logic, note that we are either a primary with - * archive_mode set to "on" or "always", or a standby with archive_mode - * set to "always". + * archive_mode set to "on" or "always", a standby with archive_mode set + * to "always", or a standby with archive_mode set to "shared". */ /* First check for .done --- this means archiver is done with it */ diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 7901c7ecc8c..4fb358ec683 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -2813,10 +2813,13 @@ WalSndArchivalReport(void) return; last_archival_report_timestamp = now; /* - * Get archiver statistics. We use non-blocking access to avoid delaying - * replication if stats collector is slow. If stats are unavailable or - * stale, we'll just try again at the next interval. + * Get archiver statistics. The pgstat snapshot is cached per-session and + * is only invalidated at transaction boundaries. The walsender runs + * without transaction boundaries, so we must clear the snapshot explicitly + * to avoid reading stale data (e.g. last_archived_wal stuck at its initial + * empty value even after the archiver has archived new segments). */ + pgstat_clear_snapshot(); archiver_stats = pgstat_fetch_stat_archiver(); if (archiver_stats == NULL) return; diff --git a/src/test/recovery/t/050_archive_shared.pl b/src/test/recovery/t/050_archive_shared.pl index 397b71ad79d..fdcdbb0657b 100644 --- a/src/test/recovery/t/050_archive_shared.pl +++ b/src/test/recovery/t/050_archive_shared.pl @@ -267,4 +267,142 @@ ok($standby2_count >= 3500, "standby2 has all data (got $standby2_count rows)"); ok($standby3_count >= 3500, "standby3 has all data (got $standby3_count rows)"); +############################################################################### +# Test 5: checkpoint on standby must NOT delete WAL that has .ready status +# +# In archive_mode=shared, the standby relies on archival reports from the +# primary to know when a segment is safe to delete. Segments not yet +# confirmed as archived have .ready files. A checkpoint (CreateRestartPoint) +# must not remove those WAL files because they may be needed for recovery +# after a standby promotion if the primary never archived them. +# +# Root cause: XLogArchiveCheckDone() treats archive_mode=shared the same as +# archive_mode=on during recovery, bypassing the .ready/.done check. +############################################################################### + +note("Test 5: checkpoint must not delete WAL with .ready on standby"); + +my $archive_dir5 = PostgreSQL::Test::Utils::tempdir(); +my $primary5 = PostgreSQL::Test::Cluster->new('primary5'); +$primary5->init(has_archiving => 1, allows_streaming => 1); +$primary5->append_conf( + 'postgresql.conf', qq{ +archive_mode = shared +archive_command = 'cp %p "$archive_dir5/%f"' +}); +$primary5->start; +$primary5->safe_psql('postgres', 'CREATE TABLE t5 (i int);'); + +# Ensure WAL activity exists in the current segment before switching. +# pg_switch_wal() is a no-op when called at the very start of a segment, +# so we write a row first to guarantee there is WAL to switch away from. +$primary5->safe_psql('postgres', 'INSERT INTO t5 VALUES (0);'); +$primary5->safe_psql('postgres', 'SELECT pg_switch_wal();'); + +# Wait for archiver to archive the switched segment +$primary5->poll_query_until('postgres', + 'SELECT archived_count > 0 FROM pg_stat_archiver') + or die "primary5: archiver did not start"; + +# Create standby without wal_keep_size so checkpoint is free to recycle segments +# backup() returns an empty list (bare "return"), so the backup name must be +# stored separately before passing it to init_from_backup. +$primary5->backup('backup5'); +my $standby5 = PostgreSQL::Test::Cluster->new('standby5'); +$standby5->init_from_backup($primary5, 'backup5', has_streaming => 1); +$standby5->append_conf( + 'postgresql.conf', qq{ +archive_mode = shared +archive_command = 'cp %p "$archive_dir5/%f"' +wal_receiver_status_interval = 1s +}); +$standby5->start; +$primary5->wait_for_catchup($standby5); + +# Break archiving on primary: new segments received by standby will get .ready +$primary5->adjust_conf('postgresql.conf', 'archive_command', "'/bin/false'"); +$primary5->reload; + +# Generate several complete WAL segments. After the standby replays all of +# them its redo pointer is well past the first few, making those candidates +# for checkpoint removal. +for (1 .. 6) +{ + $primary5->safe_psql('postgres', + 'INSERT INTO t5 SELECT generate_series(1,1000);'); + $primary5->safe_psql('postgres', 'SELECT pg_switch_wal();'); +} +$primary5->wait_for_catchup($standby5); + +# Collect every WAL segment that has a .ready file on the standby +my $status_dir5 = $standby5->data_dir . '/pg_wal/archive_status'; +my @ready5; +if (opendir(my $dh, $status_dir5)) +{ + @ready5 = map { s/\.ready$//r } grep { /\.ready$/ } readdir($dh); + closedir($dh); +} +my $n_ready5 = scalar @ready5; +note("Before checkpoint: $n_ready5 WAL files with .ready"); +cmp_ok($n_ready5, '>', 0, "standby has .ready WAL files before checkpoint"); + +# Trigger CreateRestartPoint (the standby equivalent of CHECKPOINT). +# It must not remove WAL files that carry a .ready status. +$standby5->safe_psql('postgres', 'CHECKPOINT'); + +my $wal_dir5 = $standby5->data_dir . '/pg_wal'; +my $deleted5 = 0; +for my $f (@ready5) +{ + unless (-f "$wal_dir5/$f") + { + $deleted5++; + diag("BUG: $f had .ready but checkpoint deleted it from standby"); + } +} +is($deleted5, 0, + "checkpoint does not delete WAL with .ready (not yet archived by primary)"); + +############################################################################### +# Test 6: after archiving is restored on primary, standby .ready -> .done +# +# When archive_command is broken for a while and then fixed, the primary will +# archive the previously-failed segments. The walsender sends an archival +# status report to the standby which then converts .ready to .done. +# This verifies the end-to-end recovery of the mechanism after an outage. +############################################################################### + +note("Test 6: .ready files become .done after archiving restored on primary"); + +# Capture archived_count before restoring so we can detect new archival +my $archived_before5 = + $primary5->safe_psql('postgres', 'SELECT archived_count FROM pg_stat_archiver'); + +# Restore archiving +$primary5->adjust_conf('postgresql.conf', 'archive_command', + qq{'cp %p "$archive_dir5/%f"'}); +$primary5->reload; + +# Wait for primary to archive the segments that failed during the outage +$primary5->poll_query_until('postgres', + "SELECT archived_count > $archived_before5 FROM pg_stat_archiver") + or die "primary5: archiver did not catch up after archive_command restored"; + +# The walsender sends archival status reports every ~10 s. Wait up to +# timeout_default seconds for every .ready file to transition to .done. +my $remaining5 = $n_ready5; +for (my $i = 0; $i < $PostgreSQL::Test::Utils::timeout_default; $i++) +{ + $remaining5 = 0; + if (opendir(my $dh, $status_dir5)) + { + $remaining5 = scalar(grep { /\.ready$/ } readdir($dh)); + closedir($dh); + } + last if $remaining5 == 0; + sleep(1); +} +is($remaining5, 0, + "all .ready files become .done after archiving restored on primary"); + done_testing(); diff --git a/src/test/recovery/t/051_archive_shared_checkpoint.pl b/src/test/recovery/t/051_archive_shared_checkpoint.pl new file mode 100644 index 00000000000..278fc2f8b70 --- /dev/null +++ b/src/test/recovery/t/051_archive_shared_checkpoint.pl @@ -0,0 +1,211 @@ +# Copyright (c) 2025, PostgreSQL Global Development Group + +# Tests for archive_mode=shared correctness on standbys: +# +# 1. Checkpoint on standby must NOT remove WAL segments that have a .ready +# status file (i.e. not yet archived by the primary). With the bug, +# XLogArchiveCheckDone() returns true unconditionally during recovery for +# any mode that is not "always", so checkpoint deletes these segments. +# +# 2. After archiving is broken on the primary and then restored, .ready files +# on the standby must eventually transition to .done (primary sends archival +# status reports to the standby via the walsender). + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +# Use 1 MB WAL segments so we can generate many segments cheaply. +my $wal_segsize = 1; + +# An archive command that always fails (but is recognized by the archiver as a +# real failure, not a missing command). Mirrors the approach in +# 020_archive_status.pl to stay portable. +my $broken_command = + $PostgreSQL::Test::Utils::windows_os + ? q{copy "%p_does_not_exist" "%f_does_not_exist"} + : q{cp "%p_does_not_exist" "%f_does_not_exist"}; + +my $archive_dir = PostgreSQL::Test::Utils::tempdir(); +my $good_command = + $PostgreSQL::Test::Utils::windows_os + ? qq{copy "%p" "$archive_dir\\%f"} + : qq{cp %p "$archive_dir/%f"}; + +############################################################################### +# Set up primary with archive_mode=shared and BROKEN archiving so that every +# WAL segment received by the standby gets a .ready file. +############################################################################### + +my $primary = PostgreSQL::Test::Cluster->new('primary'); +$primary->init( + has_archiving => 1, + allows_streaming => 1, + extra => [ '--wal-segsize' => $wal_segsize ]); +$primary->append_conf('postgresql.conf', qq{ +archive_mode = shared +archive_command = '$broken_command' +wal_keep_size = 0 +}); +$primary->start; + +my $backup_name = 'standby_backup'; +$primary->backup($backup_name); + +my $standby = PostgreSQL::Test::Cluster->new('standby'); +$standby->init_from_backup($primary, $backup_name, has_streaming => 1); +$standby->append_conf('postgresql.conf', qq{ +archive_mode = shared +archive_command = '$good_command' +wal_receiver_status_interval = 1s +wal_keep_size = 0 +}); +$standby->start; + +$primary->wait_for_catchup($standby); + +############################################################################### +# Generate WAL while archiving is broken. +# The walreceiver will create .ready files for every received segment. +############################################################################### + +$primary->safe_psql('postgres', 'CREATE TABLE t (x int)'); + +# Switch WAL several times to create clearly-identifiable old segments. +# We capture the name of the first switched-away segment; it is the primary +# candidate that checkpoint would delete. +my $target_seg = $primary->safe_psql('postgres', + q{SELECT pg_walfile_name(pg_current_wal_lsn())}); + +for my $i (1..5) +{ + $primary->safe_psql('postgres', + "INSERT INTO t SELECT generate_series(1,500)"); + $primary->safe_psql('postgres', 'SELECT pg_switch_wal()'); +} + +# Wait for the archiver to register failures so we are sure archiving is +# truly broken (not just slow). +$primary->poll_query_until('postgres', + q{SELECT failed_count > 0 FROM pg_stat_archiver}) + or die "Timed out waiting for archiver to fail"; + +# Issue a CHECKPOINT on the primary so that the standby can form a +# restartpoint whose redo LSN is past $target_seg. +$primary->safe_psql('postgres', 'CHECKPOINT'); + +# Wait for the standby to replay everything up to that checkpoint. +$primary->wait_for_catchup($standby); + +my $standby_wal_dir = $standby->data_dir . '/pg_wal'; +my $standby_status_dir = "$standby_wal_dir/archive_status"; + +# The target segment must already be visible on the standby as .ready. +my $target_ready = "$standby_status_dir/$target_seg.ready"; +ok(-f $target_ready, + "standby has .ready file for segment $target_seg (not archived by primary)"); + +# The WAL file itself must also be present. +ok(-f "$standby_wal_dir/$target_seg", + "WAL segment $target_seg exists in standby pg_wal before CHECKPOINT"); + +############################################################################### +# Test 1: CHECKPOINT (restartpoint) on standby must not remove .ready segments +############################################################################### + +# This triggers CreateRestartPoint, which calls RemoveOldXlogFiles. +# With the bug, XLogArchiveCheckDone returns true for every segment in +# archive_mode=shared during recovery, so $target_seg would be deleted. +$standby->safe_psql('postgres', 'CHECKPOINT'); + +ok(-f "$standby_wal_dir/$target_seg", + "WAL segment $target_seg still exists after CHECKPOINT on standby " + . "(not deleted despite .ready status)"); + +ok(-f $target_ready, + ".ready file for $target_seg still present after CHECKPOINT on standby"); + +############################################################################### +# Test 2: Restoring archiving on primary causes .ready -> .done on standby +# +# This part is independent of Test 1: we generate fresh WAL (with archiving +# still broken) so the standby accumulates new .ready files, then restore +# archiving and verify those files become .done. +############################################################################### + +# Generate a few more segments so the standby definitely has fresh .ready files +# regardless of what checkpoint may have done above. +for my $i (1..3) +{ + $primary->safe_psql('postgres', + "INSERT INTO t SELECT generate_series(1,200)"); + $primary->safe_psql('postgres', 'SELECT pg_switch_wal()'); +} +$primary->wait_for_catchup($standby); + +# Collect all current .ready files on the standby. +my @ready_segs; +if (opendir(my $dh, $standby_status_dir)) +{ + @ready_segs = + map { (my $s = $_) =~ s/\.ready$//; $s } + grep { /\.ready$/ } readdir($dh); + closedir($dh); +} +note("Standby has " + . scalar(@ready_segs) + . " .ready segments before archiving is restored"); +cmp_ok(scalar(@ready_segs), '>', 0, + "standby has fresh .ready files for newly received unarchived segments"); + +# Restore archiving on the primary. +$primary->safe_psql('postgres', qq{ + ALTER SYSTEM SET archive_command TO '$good_command'; + SELECT pg_reload_conf(); +}); + +# Wait until primary has archived at least one segment. +$primary->poll_query_until('postgres', + q{SELECT archived_count > 0 FROM pg_stat_archiver}) + or die "Timed out waiting for primary to start archiving after restore"; + +# Generate one more WAL switch so the walsender picks up the updated +# last_archived_wal and sends a fresh archival report to the standby. +# (The walsender only sends when last_archived_wal changes and every +# ARCHIVAL_REPORT_INTERVAL = 10 s at most.) +$primary->safe_psql('postgres', 'SELECT pg_switch_wal()'); +$primary->wait_for_catchup($standby); + +# Poll until all previously-.ready segments have become .done. +# Allow up to the framework default timeout (usually 120 s); the walsender +# reports every 10 s so convergence should happen well within that. +my $remaining_ready = scalar(@ready_segs); +for my $i (1 .. $PostgreSQL::Test::Utils::timeout_default) +{ + $remaining_ready = 0; + if (opendir(my $dh, $standby_status_dir)) + { + # Count only the segments that were .ready before archiving was restored + for my $seg (@ready_segs) + { + $remaining_ready++ if -f "$standby_status_dir/$seg.ready"; + } + closedir($dh); + } + last if $remaining_ready == 0; + sleep(1); +} + +is($remaining_ready, 0, + "all .ready files on standby transitioned to .done " + . "after archiving restored on primary"); + +# Sanity-check: the WAL files are still present (they weren't deleted by +# checkpoint while .ready, nor disappeared otherwise). +my @still_missing = grep { !-f "$standby_wal_dir/$_" } @ready_segs; +is(scalar(@still_missing), 0, + "WAL segments were not lost while waiting for archival reports"); + +done_testing();