From cccfeb0070383111d03d66c4f5786b0571d24feb Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Wed, 20 Aug 2025 12:57:54 -0700 Subject: [PATCH 01/45] [SPEC] Add FGAC enforcement instructions as part of loadTable --- open-api/rest-catalog-open-api.py | 22 ++++++++++++++++++++++ open-api/rest-catalog-open-api.yaml | 24 ++++++++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index 99fe855147c8..2affdcf5c1d8 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -1526,6 +1526,25 @@ class AddSchemaUpdate(BaseUpdate): ) +class FineGrainedDataProtectionRules(BaseModel): + """ + Fine-grained data protection rules for a table as result of fine grained policy evaluation at the catalog end based on the clients access rights. + The client SHOULD use these rules to enforce fine-grained data protection like column and row level access when reading data from the table. + + """ + + projections: Optional[List[Term]] = Field( + None, + description='This field contains a list of columns or column transforms to be projected.\nNote: That each column must have only a single projection, meaning a column can be projected as-is or as a transformed value, but not both.\n', + ) + row_filter: Optional[Expression] = Field( + None, + alias='row-filter', + description='An expression that filters rows. Only rows for which the expression evaluates to true should be allowed to read. If the catalog supports multiple row access filter against the table, its the catalogs responsibility to combine them with the appropriate logic (e.g., AND, OR).\n', + ) + + + class LoadTableResult(BaseModel): """ Result used when a table is successfully loaded. @@ -1580,6 +1599,9 @@ class LoadTableResult(BaseModel): storage_credentials: list[StorageCredential] | None = Field( None, alias='storage-credentials' ) + fine_grained_data_protection_instructions: Optional[ + FineGrainedDataProtectionRules + ] = Field(None, alias='fine-grained-data-protection-instructions') class ScanTasks(BaseModel): diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index b5fe7f69e37d..519538f9e22d 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3663,6 +3663,28 @@ components: additionalProperties: type: string + FineGrainedDataProtectionRules: + type: object + description: > + Fine-grained data protection rules for a table as result of fine grained policy evaluation at the catalog end based on the clients access rights. + + The client SHOULD use these rules to enforce fine-grained data protection like column and row level access when reading data from the table. + properties: + projections: + description: > + This field contains a list of columns or column transforms to be projected. + + Note: That each column must have only a single projection, meaning a column can be projected as-is or as a transformed value, but not both. + type: array + items: + $ref: '#/components/schemas/Term' + row-filter: + description: > + An expression that filters rows. Only rows for which the expression evaluates to true should be allowed to read. + If the catalog supports multiple row access filter against the table, its the catalogs responsibility to combine them with + the appropriate logic (e.g., AND, OR). + $ref: '#/components/schemas/Expression' + LoadCredentialsResponse: type: object required: @@ -3732,6 +3754,8 @@ components: type: array items: $ref: '#/components/schemas/StorageCredential' + fine-grained-data-protection-instructions: + $ref: '#/components/schemas/FineGrainedDataProtectionRules' ScanTasks: type: object From c2900efe86a49b9618487f83f6f49135bbb395b1 Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Wed, 20 Aug 2025 19:08:34 -0700 Subject: [PATCH 02/45] Address review feedbacks --- open-api/rest-catalog-open-api.py | 22 +++++++++++----------- open-api/rest-catalog-open-api.yaml | 22 +++++++++++----------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index 2affdcf5c1d8..bb10d54c4327 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -1526,21 +1526,21 @@ class AddSchemaUpdate(BaseUpdate): ) -class FineGrainedDataProtectionRules(BaseModel): +class ReadRestrictions(BaseModel): """ - Fine-grained data protection rules for a table as result of fine grained policy evaluation at the catalog end based on the clients access rights. - The client SHOULD use these rules to enforce fine-grained data protection like column and row level access when reading data from the table. + Read Restrictions for a table including projection and row filter expressions. The client MUST enforce these rules to read data from the table. """ - projections: Optional[List[Term]] = Field( + required_projection: Optional[List[Term]] = Field( None, - description='This field contains a list of columns or column transforms to be projected.\nNote: That each column must have only a single projection, meaning a column can be projected as-is or as a transformed value, but not both.\n', + alias='required-projection', + description='A list of projections that must be applied before query projections. If the term is a transform, it must replace the column referenced by the term. Readers are not allowed to project columns that are not listed and must apply transforms.\nNote: That each column must have only a single projection, meaning a column can be projected as-is or as a transformed value, but not both.\n', ) - row_filter: Optional[Expression] = Field( + required_row_filter: Optional[Expression] = Field( None, - alias='row-filter', - description='An expression that filters rows. Only rows for which the expression evaluates to true should be allowed to read. If the catalog supports multiple row access filter against the table, its the catalogs responsibility to combine them with the appropriate logic (e.g., AND, OR).\n', + alias='required-row-filter', + description='An expression that filters rows. Rows for which the filter evaluates to false must be discarded and no information derived from the filtered rows may be included in the query result. If the catalog supports multiple row access filter against the table, it is the catalogs responsibility to combine them with the appropriate logic (e.g., AND, OR).\n', ) @@ -1599,9 +1599,9 @@ class LoadTableResult(BaseModel): storage_credentials: list[StorageCredential] | None = Field( None, alias='storage-credentials' ) - fine_grained_data_protection_instructions: Optional[ - FineGrainedDataProtectionRules - ] = Field(None, alias='fine-grained-data-protection-instructions') + read_restrictions: Optional[ReadRestrictions] = Field( + None, alias='read-restrictions' + ) class ScanTasks(BaseModel): diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index 519538f9e22d..344f3f401845 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3663,25 +3663,25 @@ components: additionalProperties: type: string - FineGrainedDataProtectionRules: + ReadRestrictions: type: object description: > - Fine-grained data protection rules for a table as result of fine grained policy evaluation at the catalog end based on the clients access rights. - - The client SHOULD use these rules to enforce fine-grained data protection like column and row level access when reading data from the table. + Read Restrictions for a table including projection and row filter expressions. + The client MUST enforce these rules to read data from the table. properties: - projections: + required-projection: description: > - This field contains a list of columns or column transforms to be projected. + A list of projections that must be applied before query projections. If the term is a transform, it must replace the column referenced by the term. + Readers are not allowed to project columns that are not listed and must apply transforms. Note: That each column must have only a single projection, meaning a column can be projected as-is or as a transformed value, but not both. type: array items: $ref: '#/components/schemas/Term' - row-filter: + required-row-filter: description: > - An expression that filters rows. Only rows for which the expression evaluates to true should be allowed to read. - If the catalog supports multiple row access filter against the table, its the catalogs responsibility to combine them with + An expression that filters rows. Rows for which the filter evaluates to false must be discarded and no information derived from the filtered rows may be included in the query result. + If the catalog supports multiple row access filter against the table, it is the catalogs responsibility to combine them with the appropriate logic (e.g., AND, OR). $ref: '#/components/schemas/Expression' @@ -3754,8 +3754,8 @@ components: type: array items: $ref: '#/components/schemas/StorageCredential' - fine-grained-data-protection-instructions: - $ref: '#/components/schemas/FineGrainedDataProtectionRules' + read-restrictions: + $ref: '#/components/schemas/ReadRestrictions' ScanTasks: type: object From 4c567a74cf3aa5fb5451b5546f1222c8600bb9a1 Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Thu, 21 Aug 2025 14:32:37 -0700 Subject: [PATCH 03/45] Address review feedbacks --- open-api/rest-catalog-open-api.py | 6 +++--- open-api/rest-catalog-open-api.yaml | 7 ++++++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index bb10d54c4327..02a764f32c20 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -1528,19 +1528,19 @@ class AddSchemaUpdate(BaseUpdate): class ReadRestrictions(BaseModel): """ - Read Restrictions for a table including projection and row filter expressions. The client MUST enforce these rules to read data from the table. + Read Restrictions for a table including projection and row filter expressions. The client MUST enforce these rules to read data from the table. If the read-restrictions section is not present or is empty, clients MUST treat it as equivalent to having no restrictions. """ required_projection: Optional[List[Term]] = Field( None, alias='required-projection', - description='A list of projections that must be applied before query projections. If the term is a transform, it must replace the column referenced by the term. Readers are not allowed to project columns that are not listed and must apply transforms.\nNote: That each column must have only a single projection, meaning a column can be projected as-is or as a transformed value, but not both.\n', + description='A list of projections that must be applied before query projections. If the term is a transform, it must replace the column referenced by the term. For example, if the term is mask(cc, 0, 4) i.e mask transform on column cc, it must replace the column cc in the query with the masked value, essentially projecting it as mask(cc, 0, 4) AS cc. Readers are NOT allowed to project columns that are not listed and must apply transforms. If the required-projection is not present or is empty, it means that no projection is required and all columns can be read as-is.\nNote: That each column must have only a single projection, meaning a column can be projected as-is or as a transformed value, but not both.\n', ) required_row_filter: Optional[Expression] = Field( None, alias='required-row-filter', - description='An expression that filters rows. Rows for which the filter evaluates to false must be discarded and no information derived from the filtered rows may be included in the query result. If the catalog supports multiple row access filter against the table, it is the catalogs responsibility to combine them with the appropriate logic (e.g., AND, OR).\n', + description='An expression that filters rows. Rows for which the filter evaluates to false must be discarded and no information derived from the filtered rows may be included in the query result. If the catalog supports multiple row access filter against the table, it is the catalogs responsibility to combine them with the appropriate logic (e.g., AND, OR). If the required-row-filter is not present or is empty, it means that no row filtering is required and all rows can be read.\n', ) diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index 344f3f401845..e1e07bda0093 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3668,11 +3668,15 @@ components: description: > Read Restrictions for a table including projection and row filter expressions. The client MUST enforce these rules to read data from the table. + If the read-restrictions section is not present or is empty, clients MUST treat it as equivalent to having no restrictions. properties: required-projection: description: > A list of projections that must be applied before query projections. If the term is a transform, it must replace the column referenced by the term. - Readers are not allowed to project columns that are not listed and must apply transforms. + For example, if the term is mask(cc, 0, 4) i.e mask transform on column cc, it must replace the column cc in the query with the masked value, + essentially projecting it as mask(cc, 0, 4) AS cc. + Readers are NOT allowed to project columns that are not listed and must apply transforms. + If the required-projection is not present or is empty, it means that no projection is required and all columns can be read as-is. Note: That each column must have only a single projection, meaning a column can be projected as-is or as a transformed value, but not both. type: array @@ -3683,6 +3687,7 @@ components: An expression that filters rows. Rows for which the filter evaluates to false must be discarded and no information derived from the filtered rows may be included in the query result. If the catalog supports multiple row access filter against the table, it is the catalogs responsibility to combine them with the appropriate logic (e.g., AND, OR). + If the required-row-filter is not present or is empty, it means that no row filtering is required and all rows can be read. $ref: '#/components/schemas/Expression' LoadCredentialsResponse: From 0138cbf60fa24b0bb00d9fa86582197ba79abda8 Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Fri, 22 Aug 2025 15:55:50 -0700 Subject: [PATCH 04/45] Address review feedback --- open-api/rest-catalog-open-api.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index e1e07bda0093..4b1566b0d84b 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3673,8 +3673,8 @@ components: required-projection: description: > A list of projections that must be applied before query projections. If the term is a transform, it must replace the column referenced by the term. - For example, if the term is mask(cc, 0, 4) i.e mask transform on column cc, it must replace the column cc in the query with the masked value, - essentially projecting it as mask(cc, 0, 4) AS cc. + For example, if the term is truncate(4, cc) i.e truncate transform on column cc, it must replace the column cc in the query with the transformed value, + essentially projecting it as truncate(4, cc) AS cc. Readers are NOT allowed to project columns that are not listed and must apply transforms. If the required-projection is not present or is empty, it means that no projection is required and all columns can be read as-is. From 65b88d3b7ac3c9444e53934ccceffe4635bc21e1 Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Mon, 25 Aug 2025 16:12:20 -0700 Subject: [PATCH 05/45] Address review feedbacks and align to RFC-2119 --- open-api/rest-catalog-open-api.py | 6 ++-- open-api/rest-catalog-open-api.yaml | 53 ++++++++++++++++++++++------- 2 files changed, 44 insertions(+), 15 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index 02a764f32c20..3c4d9355b736 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -1528,19 +1528,19 @@ class AddSchemaUpdate(BaseUpdate): class ReadRestrictions(BaseModel): """ - Read Restrictions for a table including projection and row filter expressions. The client MUST enforce these rules to read data from the table. If the read-restrictions section is not present or is empty, clients MUST treat it as equivalent to having no restrictions. + Read Restrictions for a table including projection and row filter expressions. The client MUST enforce these rules to read data from the table. """ required_projection: Optional[List[Term]] = Field( None, alias='required-projection', - description='A list of projections that must be applied before query projections. If the term is a transform, it must replace the column referenced by the term. For example, if the term is mask(cc, 0, 4) i.e mask transform on column cc, it must replace the column cc in the query with the masked value, essentially projecting it as mask(cc, 0, 4) AS cc. Readers are NOT allowed to project columns that are not listed and must apply transforms. If the required-projection is not present or is empty, it means that no projection is required and all columns can be read as-is.\nNote: That each column must have only a single projection, meaning a column can be projected as-is or as a transformed value, but not both.\n', + description='A list of projections that MUST be applied prior to any query-specified projections. If the required-projection property is absent or empty, no mandatory projection applies, and a reader MAY project any subset of columns of the table, including all columns.\n1. A reader MUST project only columns listed in the required-projection.\n - If a listed column has a transform, the reader MUST apply it and replace\n all references to the underlying column with the transformed value\n (for example, truncate(4, cc) MUST be projected as truncate(4, cc) AS cc,\n and all references to cc during query evaluation MUST resolve to this alias).\n - If a listed column has no transform, the reader MUST read it as-is.\n - Columns not listed in the required-projection MUST NOT be read.\n\n2. A column MUST appear at most once in the required-projection.\n3. A projection entry MUST reference either the column itself or exactly one\n transformed version of the column, but not both.\n\n4. Multiple transformed versions of the same column (e.g., truncate(5, col)\n and truncate(3, col)) MUST NOT appear in the required-projection.\n\n5. If a projection entry includes a transform that the reader cannot evaluate,\n the reader MUST fail rather than ignore the transform.\n\n6. An identity transform is equivalent to projecting the column directly.\n A reader MAY represent it in either form.\n', ) required_row_filter: Optional[Expression] = Field( None, alias='required-row-filter', - description='An expression that filters rows. Rows for which the filter evaluates to false must be discarded and no information derived from the filtered rows may be included in the query result. If the catalog supports multiple row access filter against the table, it is the catalogs responsibility to combine them with the appropriate logic (e.g., AND, OR). If the required-row-filter is not present or is empty, it means that no row filtering is required and all rows can be read.\n', + description="An expression that filters rows in the table.\n1. A reader MUST discard any row for which the filter evaluates to false, and\n no information derived from discarded rows MAY be included in the query result.\n\n2. If the catalog supports multiple row access filters for the table, it is\n the catalog's responsibility to combine them using the appropriate logic (e.g., AND, OR).\n\n3. If a client cannot interpret or evaluate a provided filter expression, it\n MUST NOT return partially filtered results and MUST fail.\n\n4. If the required-row-filter property is absent or empty, no mandatory\n filtering is imposed, and a reader MAY return any subset of rows,\n including all rows.\n", ) diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index 4b1566b0d84b..26a7f9962720 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3668,26 +3668,55 @@ components: description: > Read Restrictions for a table including projection and row filter expressions. The client MUST enforce these rules to read data from the table. - If the read-restrictions section is not present or is empty, clients MUST treat it as equivalent to having no restrictions. properties: required-projection: description: > - A list of projections that must be applied before query projections. If the term is a transform, it must replace the column referenced by the term. - For example, if the term is truncate(4, cc) i.e truncate transform on column cc, it must replace the column cc in the query with the transformed value, - essentially projecting it as truncate(4, cc) AS cc. - Readers are NOT allowed to project columns that are not listed and must apply transforms. - If the required-projection is not present or is empty, it means that no projection is required and all columns can be read as-is. - - Note: That each column must have only a single projection, meaning a column can be projected as-is or as a transformed value, but not both. + A list of projections that MUST be applied prior to any query-specified + projections. + If the required-projection property is absent or empty, no mandatory projection applies, + and a reader MAY project any subset of columns of the table, including all columns. + + 1. A reader MUST project only columns listed in the required-projection. + - If a listed column has a transform, the reader MUST apply it and replace + all references to the underlying column with the transformed value + (for example, truncate(4, cc) MUST be projected as truncate(4, cc) AS cc, + and all references to cc during query evaluation MUST resolve to this alias). + - If a listed column has no transform, the reader MUST read it as-is. + - Columns not listed in the required-projection MUST NOT be read. + + 2. A column MUST appear at most once in the required-projection. + + 3. A projection entry MUST reference either the column itself or exactly one + transformed version of the column, but not both. + + 4. Multiple transformed versions of the same column (e.g., truncate(5, col) + and truncate(3, col)) MUST NOT appear in the required-projection. + + 5. If a projection entry includes a transform that the reader cannot evaluate, + the reader MUST fail rather than ignore the transform. + + 6. An identity transform is equivalent to projecting the column directly. + A reader MAY represent it in either form. + type: array items: $ref: '#/components/schemas/Term' required-row-filter: description: > - An expression that filters rows. Rows for which the filter evaluates to false must be discarded and no information derived from the filtered rows may be included in the query result. - If the catalog supports multiple row access filter against the table, it is the catalogs responsibility to combine them with - the appropriate logic (e.g., AND, OR). - If the required-row-filter is not present or is empty, it means that no row filtering is required and all rows can be read. + An expression that filters rows in the table. + + 1. A reader MUST discard any row for which the filter evaluates to false, and + no information derived from discarded rows MAY be included in the query result. + + 2. If the catalog supports multiple row access filters for the table, it is + the catalog's responsibility to combine them using the appropriate logic (e.g., AND, OR). + + 3. If a client cannot interpret or evaluate a provided filter expression, it + MUST NOT return partially filtered results and MUST fail. + + 4. If the required-row-filter property is absent or empty, no mandatory + filtering is imposed, and a reader MAY return any subset of rows, + including all rows. $ref: '#/components/schemas/Expression' LoadCredentialsResponse: From 1e8f75c78bca1725f13c0ccb2e27260045eeaa0f Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Tue, 26 Aug 2025 08:36:57 -0700 Subject: [PATCH 06/45] Make who the evaluated policy is applicable to more verbose --- open-api/rest-catalog-open-api.py | 4 +++- open-api/rest-catalog-open-api.yaml | 11 +++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index 3c4d9355b736..ee35ce3eb67e 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -1528,7 +1528,9 @@ class AddSchemaUpdate(BaseUpdate): class ReadRestrictions(BaseModel): """ - Read Restrictions for a table including projection and row filter expressions. The client MUST enforce these rules to read data from the table. + Read restrictions for a table, including projection and row filter expressions. + A client MUST enforce the restrictions defined in this object when reading data from the table. + These restrictions apply only to the authenticated principal, user, or account associated with the client. They MUST NOT be interpreted as global policy and MUST NOT be applied beyond the entity identified by the Authentication header (or other applicable authentication mechanism). """ diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index 26a7f9962720..e686b6044393 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3666,8 +3666,15 @@ components: ReadRestrictions: type: object description: > - Read Restrictions for a table including projection and row filter expressions. - The client MUST enforce these rules to read data from the table. + Read restrictions for a table, including projection and row filter expressions. + + A client MUST enforce the restrictions defined in this object when reading data + from the table. + + These restrictions apply only to the authenticated principal, user, or account + associated with the client. They MUST NOT be interpreted as global policy and + MUST NOT be applied beyond the entity identified by the Authentication header + (or other applicable authentication mechanism). properties: required-projection: description: > From ac7ad1f47b9205e1938ea2adb2b6708f8237457b Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Wed, 27 Aug 2025 11:49:50 -0700 Subject: [PATCH 07/45] Add projection must be applied after filtering --- open-api/rest-catalog-open-api.py | 2 +- open-api/rest-catalog-open-api.yaml | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index ee35ce3eb67e..5bff872169fb 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -1542,7 +1542,7 @@ class ReadRestrictions(BaseModel): required_row_filter: Optional[Expression] = Field( None, alias='required-row-filter', - description="An expression that filters rows in the table.\n1. A reader MUST discard any row for which the filter evaluates to false, and\n no information derived from discarded rows MAY be included in the query result.\n\n2. If the catalog supports multiple row access filters for the table, it is\n the catalog's responsibility to combine them using the appropriate logic (e.g., AND, OR).\n\n3. If a client cannot interpret or evaluate a provided filter expression, it\n MUST NOT return partially filtered results and MUST fail.\n\n4. If the required-row-filter property is absent or empty, no mandatory\n filtering is imposed, and a reader MAY return any subset of rows,\n including all rows.\n", + description="An expression that filters rows in the table.\n1. A reader MUST discard any row for which the filter evaluates to false, and\n no information derived from discarded rows MAY be included in the query result.\n\n2. Row filters MUST be evaluated against the original, untransformed column values.\n Required projections MUST be applied only after row filters are applied.\n\n2. If the catalog supports multiple row access filters for the table, it is\n the catalog's responsibility to combine them using the appropriate logic (e.g., AND, OR).\n\n3. If a client cannot interpret or evaluate a provided filter expression, it\n MUST NOT return partially filtered results and MUST fail.\n\n4. If the required-row-filter property is absent or empty, no mandatory\n filtering is imposed, and a reader MAY return any subset of rows,\n including all rows.\n", ) diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index e686b6044393..aa0028de8690 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3714,6 +3714,9 @@ components: 1. A reader MUST discard any row for which the filter evaluates to false, and no information derived from discarded rows MAY be included in the query result. + + 2. Row filters MUST be evaluated against the original, untransformed column values. + Required projections MUST be applied only after row filters are applied. 2. If the catalog supports multiple row access filters for the table, it is the catalog's responsibility to combine them using the appropriate logic (e.g., AND, OR). From 239ce1606b5e6bd5ba8cf50d49c29d411a8c0a1c Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Wed, 27 Aug 2025 18:31:08 -0700 Subject: [PATCH 08/45] Add restriction for data type match --- open-api/rest-catalog-open-api.py | 2 +- open-api/rest-catalog-open-api.yaml | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index 5bff872169fb..a7accfe85f82 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -1537,7 +1537,7 @@ class ReadRestrictions(BaseModel): required_projection: Optional[List[Term]] = Field( None, alias='required-projection', - description='A list of projections that MUST be applied prior to any query-specified projections. If the required-projection property is absent or empty, no mandatory projection applies, and a reader MAY project any subset of columns of the table, including all columns.\n1. A reader MUST project only columns listed in the required-projection.\n - If a listed column has a transform, the reader MUST apply it and replace\n all references to the underlying column with the transformed value\n (for example, truncate(4, cc) MUST be projected as truncate(4, cc) AS cc,\n and all references to cc during query evaluation MUST resolve to this alias).\n - If a listed column has no transform, the reader MUST read it as-is.\n - Columns not listed in the required-projection MUST NOT be read.\n\n2. A column MUST appear at most once in the required-projection.\n3. A projection entry MUST reference either the column itself or exactly one\n transformed version of the column, but not both.\n\n4. Multiple transformed versions of the same column (e.g., truncate(5, col)\n and truncate(3, col)) MUST NOT appear in the required-projection.\n\n5. If a projection entry includes a transform that the reader cannot evaluate,\n the reader MUST fail rather than ignore the transform.\n\n6. An identity transform is equivalent to projecting the column directly.\n A reader MAY represent it in either form.\n', + description='A list of projections that MUST be applied prior to any query-specified projections. If the required-projection property is absent or empty, no mandatory projection applies, and a reader MAY project any subset of columns of the table, including all columns.\n1. A reader MUST project only columns listed in the required-projection.\n - If a listed column has a transform, the reader MUST apply it and replace\n all references to the underlying column with the transformed value\n (for example, truncate(4, cc) MUST be projected as truncate(4, cc) AS cc,\n and all references to cc during query evaluation MUST resolve to this alias).\n - If a listed column has no transform, the reader MUST read it as-is.\n - Columns not listed in the required-projection MUST NOT be read.\n\n2. A column MUST appear at most once in the required-projection.\n3. A projection entry MUST reference either the column itself or exactly one\n transformed version of the column, but not both.\n\n4. Multiple transformed versions of the same column (e.g., truncate(5, col)\n and truncate(3, col)) MUST NOT appear in the required-projection.\n\n5. If a projection entry includes a transform that the reader cannot evaluate,\n the reader MUST fail rather than ignore the transform.\n\n6. An identity transform is equivalent to projecting the column directly.\n A reader MAY represent it in either form.\n\n7. The data type of the projected column MUST match the data type defined for the transform result.\n', ) required_row_filter: Optional[Expression] = Field( None, diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index aa0028de8690..df639c2eef0b 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3704,6 +3704,8 @@ components: 6. An identity transform is equivalent to projecting the column directly. A reader MAY represent it in either form. + + 7. The data type of the projected column MUST match the data type defined for the transform result. type: array items: From 6cb36e4a2430b04a656b3eb864a50297c9bfecd0 Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Sun, 7 Sep 2025 23:25:09 -0700 Subject: [PATCH 09/45] [Ongoing Discussion] Add projections and filters are per current schema for now --- open-api/rest-catalog-open-api.py | 2 +- open-api/rest-catalog-open-api.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index a7accfe85f82..22436434484c 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -1528,7 +1528,7 @@ class AddSchemaUpdate(BaseUpdate): class ReadRestrictions(BaseModel): """ - Read restrictions for a table, including projection and row filter expressions. + Read restrictions for a table, including projection and row filter expressions, according to the current schema. A client MUST enforce the restrictions defined in this object when reading data from the table. These restrictions apply only to the authenticated principal, user, or account associated with the client. They MUST NOT be interpreted as global policy and MUST NOT be applied beyond the entity identified by the Authentication header (or other applicable authentication mechanism). diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index df639c2eef0b..91cd21ac1319 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3666,7 +3666,7 @@ components: ReadRestrictions: type: object description: > - Read restrictions for a table, including projection and row filter expressions. + Read restrictions for a table, including projection and row filter expressions, according to the current schema. A client MUST enforce the restrictions defined in this object when reading data from the table. From da91571a3d1ae5b1dc642edb5936a8221ce1b139 Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Wed, 24 Sep 2025 18:56:32 -0700 Subject: [PATCH 10/45] Address review feedbacks --- open-api/rest-catalog-open-api.py | 2 +- open-api/rest-catalog-open-api.yaml | 9 +++------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index 22436434484c..ff2dceff40bd 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -1542,7 +1542,7 @@ class ReadRestrictions(BaseModel): required_row_filter: Optional[Expression] = Field( None, alias='required-row-filter', - description="An expression that filters rows in the table.\n1. A reader MUST discard any row for which the filter evaluates to false, and\n no information derived from discarded rows MAY be included in the query result.\n\n2. Row filters MUST be evaluated against the original, untransformed column values.\n Required projections MUST be applied only after row filters are applied.\n\n2. If the catalog supports multiple row access filters for the table, it is\n the catalog's responsibility to combine them using the appropriate logic (e.g., AND, OR).\n\n3. If a client cannot interpret or evaluate a provided filter expression, it\n MUST NOT return partially filtered results and MUST fail.\n\n4. If the required-row-filter property is absent or empty, no mandatory\n filtering is imposed, and a reader MAY return any subset of rows,\n including all rows.\n", + description="An expression that filters rows in the table.\n1. A reader MUST discard any row for which the filter evaluates to false, and\n no information derived from discarded rows MAY be included in the query result.\n\n2. Row filters MUST be evaluated against the original, untransformed column values.\n Required projections MUST be applied only after row filters are applied.\n\n3. If the catalog supports multiple row access filters for the table, it is\n the catalog's responsibility to combine them using the appropriate logic (e.g., AND, OR).\n\n4. If a client cannot interpret or evaluate a provided filter expression, it MUST fail.\n5. If the required-row-filter property is absent or empty, no mandatory filtering is imposed.\n", ) diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index 91cd21ac1319..76e366c47c09 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3720,15 +3720,12 @@ components: 2. Row filters MUST be evaluated against the original, untransformed column values. Required projections MUST be applied only after row filters are applied. - 2. If the catalog supports multiple row access filters for the table, it is + 3. If the catalog supports multiple row access filters for the table, it is the catalog's responsibility to combine them using the appropriate logic (e.g., AND, OR). - 3. If a client cannot interpret or evaluate a provided filter expression, it - MUST NOT return partially filtered results and MUST fail. + 4. If a client cannot interpret or evaluate a provided filter expression, it MUST fail. - 4. If the required-row-filter property is absent or empty, no mandatory - filtering is imposed, and a reader MAY return any subset of rows, - including all rows. + 5. If the required-row-filter property is absent or empty, no mandatory filtering is imposed. $ref: '#/components/schemas/Expression' LoadCredentialsResponse: From 7f0a6e5192ae3e2d00206a2a88a95c44c4d01a6f Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Thu, 30 Oct 2025 06:00:36 -0700 Subject: [PATCH 11/45] Address review feedback propose a new structure` --- open-api/rest-catalog-open-api.py | 66 +++++++++++++++++-- open-api/rest-catalog-open-api.yaml | 99 ++++++++++++++++++++++++----- 2 files changed, 143 insertions(+), 22 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index ff2dceff40bd..4a4e276673b1 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -589,6 +589,42 @@ class StorageCredential(BaseModel): config: dict[str, str] +class MaskHashSha256(BaseModel): + """ + Mask the data of the column by apply SHA256 hash algorithm. Engines are free to use their own implementation of SHA256. + """ + + type: Literal['mask_hash_sha256'] + + +class MaskReplaceWithNull(BaseModel): + """ + Masks data by replacing it with a NULL value. + """ + + type: Literal['mask_replace_with_null'] + + +class MaskAlphanumeric(BaseModel): + """ + mask all alphabetic characters with 'x' and numeric characters with 'n' + """ + + type: Literal['mask_alphanumeric'] + + +class ApplyTransform(BaseModel): + """ + Applies the transform + """ + + type: Literal['apply_transform'] + transform: Transform + source_ids: List[int] = Field( + ..., alias='source-ids', description='args of transform' + ) + + class LoadCredentialsResponse(BaseModel): storage_credentials: list[StorageCredential] = Field( ..., alias='storage-credentials' @@ -1152,6 +1188,15 @@ class ViewRequirement(RootModel[AssertViewUUID]): root: AssertViewUUID = Field(..., discriminator='type') +class Action(BaseModel): + __root__: Union[ + MaskHashSha256, MaskReplaceWithNull, MaskAlphanumeric, ApplyTransform + ] = Field( + ..., + description='Defines the specific action to be executed for computing the projection.', + ) + + class FailedPlanningResult(IcebergErrorResponse): """ Failed server-side planning result @@ -1286,6 +1331,17 @@ class FunctionDefinitionVersion(BaseModel): ) +class Projection(BaseModel): + """ + Defines a projection for a column. + """ + + source_id: Any = Field( + ..., alias='source-id', description='field id of the column being projected.' + ) + action: Action + + class UnaryExpression(BaseModel): type: Literal['is-null', 'not-null', 'is-nan', 'not-nan'] = Field( ..., @@ -1528,21 +1584,21 @@ class AddSchemaUpdate(BaseUpdate): class ReadRestrictions(BaseModel): """ - Read restrictions for a table, including projection and row filter expressions, according to the current schema. + Read restrictions for a table, including projections and row filter expressions, according to the current schema. A client MUST enforce the restrictions defined in this object when reading data from the table. These restrictions apply only to the authenticated principal, user, or account associated with the client. They MUST NOT be interpreted as global policy and MUST NOT be applied beyond the entity identified by the Authentication header (or other applicable authentication mechanism). """ - required_projection: Optional[List[Term]] = Field( + required_projections: Optional[List[Projection]] = Field( None, - alias='required-projection', - description='A list of projections that MUST be applied prior to any query-specified projections. If the required-projection property is absent or empty, no mandatory projection applies, and a reader MAY project any subset of columns of the table, including all columns.\n1. A reader MUST project only columns listed in the required-projection.\n - If a listed column has a transform, the reader MUST apply it and replace\n all references to the underlying column with the transformed value\n (for example, truncate(4, cc) MUST be projected as truncate(4, cc) AS cc,\n and all references to cc during query evaluation MUST resolve to this alias).\n - If a listed column has no transform, the reader MUST read it as-is.\n - Columns not listed in the required-projection MUST NOT be read.\n\n2. A column MUST appear at most once in the required-projection.\n3. A projection entry MUST reference either the column itself or exactly one\n transformed version of the column, but not both.\n\n4. Multiple transformed versions of the same column (e.g., truncate(5, col)\n and truncate(3, col)) MUST NOT appear in the required-projection.\n\n5. If a projection entry includes a transform that the reader cannot evaluate,\n the reader MUST fail rather than ignore the transform.\n\n6. An identity transform is equivalent to projecting the column directly.\n A reader MAY represent it in either form.\n\n7. The data type of the projected column MUST match the data type defined for the transform result.\n', + alias='required-projections', + description='A list of projections that MUST be applied prior to any query-specified projections. If the required-projection property is absent, no mandatory projection applies, and a reader MAY project any subset of columns of the table, including all columns.\n1. A reader MUST project only columns listed in the required-projection.\n - If a listed column has a transform, the reader MUST apply it and replace\n all references to the underlying column with the transformed value\n (for example, truncate[4](cc) MUST be projected as truncate[4](cc) AS cc,\n and all references to cc during query evaluation post applying required-row-filter MUST resolve to this alias).\n - Columns not listed in the required-projection MUST NOT be read.\n\n2. A column MUST appear at most once in the required-projection.\n3. Multiple transformed versions of the same column (e.g., truncate[5](col)\n and truncate[3](col) MUST NOT appear in the required-projection.\n\n4. If a projection entry includes an action that the reader cannot evaluate,\n the reader MUST fail rather than ignore the transform.\n\n5. An identity transform is equivalent to projecting the column directly.\n8. The data type of the projected column MUST match the data type defined for the transform result.\n', ) required_row_filter: Optional[Expression] = Field( None, alias='required-row-filter', - description="An expression that filters rows in the table.\n1. A reader MUST discard any row for which the filter evaluates to false, and\n no information derived from discarded rows MAY be included in the query result.\n\n2. Row filters MUST be evaluated against the original, untransformed column values.\n Required projections MUST be applied only after row filters are applied.\n\n3. If the catalog supports multiple row access filters for the table, it is\n the catalog's responsibility to combine them using the appropriate logic (e.g., AND, OR).\n\n4. If a client cannot interpret or evaluate a provided filter expression, it MUST fail.\n5. If the required-row-filter property is absent or empty, no mandatory filtering is imposed.\n", + description="An expression that filters rows in the table.\n1. A reader MUST discard any row for which the filter evaluates to false or null, and\n no information derived from discarded rows MAY be included in the query result.\n\n2. Row filters MUST be evaluated against the original, untransformed column values.\n Required projections MUST be applied only after row filters are applied.\n\n3. If the catalog supports multiple row access filters for the table, it is\n the catalog's responsibility to combine them using the appropriate logic (e.g., AND, OR).\n\n4. If a client cannot interpret or evaluate a provided filter expression, it MUST fail.\n5. If the required-row-filter property is absent or empty, no mandatory filtering is imposed.\n", ) diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index 76e366c47c09..03c9880b68db 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3666,7 +3666,7 @@ components: ReadRestrictions: type: object description: > - Read restrictions for a table, including projection and row filter expressions, according to the current schema. + Read restrictions for a table, including projections and row filter expressions, according to the current schema. A client MUST enforce the restrictions defined in this object when reading data from the table. @@ -3676,45 +3676,40 @@ components: MUST NOT be applied beyond the entity identified by the Authentication header (or other applicable authentication mechanism). properties: - required-projection: + required-projections: description: > A list of projections that MUST be applied prior to any query-specified projections. - If the required-projection property is absent or empty, no mandatory projection applies, + If the required-projection property is absent, no mandatory projection applies, and a reader MAY project any subset of columns of the table, including all columns. 1. A reader MUST project only columns listed in the required-projection. - If a listed column has a transform, the reader MUST apply it and replace all references to the underlying column with the transformed value - (for example, truncate(4, cc) MUST be projected as truncate(4, cc) AS cc, - and all references to cc during query evaluation MUST resolve to this alias). - - If a listed column has no transform, the reader MUST read it as-is. + (for example, truncate[4](cc) MUST be projected as truncate[4](cc) AS cc, + and all references to cc during query evaluation post applying required-row-filter MUST resolve to this alias). - Columns not listed in the required-projection MUST NOT be read. 2. A column MUST appear at most once in the required-projection. - 3. A projection entry MUST reference either the column itself or exactly one - transformed version of the column, but not both. + 3. Multiple transformed versions of the same column (e.g., truncate[5](col) + and truncate[3](col) MUST NOT appear in the required-projection. - 4. Multiple transformed versions of the same column (e.g., truncate(5, col) - and truncate(3, col)) MUST NOT appear in the required-projection. - - 5. If a projection entry includes a transform that the reader cannot evaluate, + 4. If a projection entry includes an action that the reader cannot evaluate, the reader MUST fail rather than ignore the transform. - 6. An identity transform is equivalent to projecting the column directly. - A reader MAY represent it in either form. + 5. An identity transform is equivalent to projecting the column directly. - 7. The data type of the projected column MUST match the data type defined for the transform result. + 8. The data type of the projected column MUST match the data type defined for the transform result. type: array items: - $ref: '#/components/schemas/Term' + $ref: '#/components/schemas/Projection' required-row-filter: description: > An expression that filters rows in the table. - 1. A reader MUST discard any row for which the filter evaluates to false, and + 1. A reader MUST discard any row for which the filter evaluates to false or null, and no information derived from discarded rows MAY be included in the query result. 2. Row filters MUST be evaluated against the original, untransformed column values. @@ -3728,6 +3723,76 @@ components: 5. If the required-row-filter property is absent or empty, no mandatory filtering is imposed. $ref: '#/components/schemas/Expression' + Projection: + type: object + description: Defines a projection for a column. + properties: + source-id: + type: int + description: field id of the column being projected. + action: + $ref: '#/components/schemas/Action' + required: + - source-id + - action + + Action: + description: Defines the specific action to be executed for computing the projection. + oneOf: + - $ref: '#/components/schemas/MaskHashSha256' + - $ref: '#/components/schemas/MaskReplaceWithNull' + - $ref: '#/components/schemas/MaskAlphanumeric' + - $ref: '#/components/schemas/ApplyTransform' + + MaskHashSha256: + type: object + description: Mask the data of the column by apply SHA256 hash algorithm. Engines are free to use their own implementation of SHA256. + properties: + type: + type: string + enum: [ mask_hash_sha256 ] + required: + - type + + MaskReplaceWithNull: + type: object + description: Masks data by replacing it with a NULL value. + properties: + type: + type: string + enum: [ mask_replace_with_null ] + required: + - type + + MaskAlphanumeric: + type: object + description: mask all alphabetic characters with 'x' and numeric characters with 'n' + properties: + type: + type: string + enum: [ mask_alphanumeric ] + required: + - type + + ApplyTransform: + type: object + description: Applies the transform + properties: + type: + type: string + enum: [ apply_transform ] + transform: + $ref: '#/components/schemas/Transform' + source-ids: + description: args of transform + type: array + items: + type: integer + required: + - type + - transform + - source-ids + LoadCredentialsResponse: type: object required: From c74993a4fab81a47bacb297c93312aea614dd529 Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Tue, 4 Nov 2025 19:43:11 -0800 Subject: [PATCH 12/45] Address review feedback --- open-api/rest-catalog-open-api.py | 12 ++++++------ open-api/rest-catalog-open-api.yaml | 23 ++++++++++------------- 2 files changed, 16 insertions(+), 19 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index 4a4e276673b1..49487f3a40d4 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -1336,8 +1336,8 @@ class Projection(BaseModel): Defines a projection for a column. """ - source_id: Any = Field( - ..., alias='source-id', description='field id of the column being projected.' + field_id: int = Field( + ..., alias='field-id', description='field id of the column being projected.' ) action: Action @@ -1584,16 +1584,16 @@ class AddSchemaUpdate(BaseUpdate): class ReadRestrictions(BaseModel): """ - Read restrictions for a table, including projections and row filter expressions, according to the current schema. + Read restrictions for a table, including column projections and row filter expressions, according to the current schema. A client MUST enforce the restrictions defined in this object when reading data from the table. These restrictions apply only to the authenticated principal, user, or account associated with the client. They MUST NOT be interpreted as global policy and MUST NOT be applied beyond the entity identified by the Authentication header (or other applicable authentication mechanism). """ - required_projections: Optional[List[Projection]] = Field( + required_column_projections: Optional[List[Projection]] = Field( None, - alias='required-projections', - description='A list of projections that MUST be applied prior to any query-specified projections. If the required-projection property is absent, no mandatory projection applies, and a reader MAY project any subset of columns of the table, including all columns.\n1. A reader MUST project only columns listed in the required-projection.\n - If a listed column has a transform, the reader MUST apply it and replace\n all references to the underlying column with the transformed value\n (for example, truncate[4](cc) MUST be projected as truncate[4](cc) AS cc,\n and all references to cc during query evaluation post applying required-row-filter MUST resolve to this alias).\n - Columns not listed in the required-projection MUST NOT be read.\n\n2. A column MUST appear at most once in the required-projection.\n3. Multiple transformed versions of the same column (e.g., truncate[5](col)\n and truncate[3](col) MUST NOT appear in the required-projection.\n\n4. If a projection entry includes an action that the reader cannot evaluate,\n the reader MUST fail rather than ignore the transform.\n\n5. An identity transform is equivalent to projecting the column directly.\n8. The data type of the projected column MUST match the data type defined for the transform result.\n', + alias='required-column-projections', + description='A list of projections that MUST be applied prior to any query-specified projections. If the required-colum-projections property is absent, no mandatory projection applies, and a reader MAY project any subset of columns of the table, including all columns.\n1. A reader MUST project only columns listed in the required-colum-projections.\n - If a listed column has a transform, the reader MUST apply it and replace\n all references to the underlying column with the transformed value\n (for example, truncate[4](cc) MUST be projected as truncate[4](cc) AS cc,\n and all references to cc during query evaluation post applying required-row-filter MUST resolve to this alias).\n - Columns not listed in the required-colum-projections MUST NOT be read.\n\n2. A column MUST appear at most once in the required-colum-projections.\n3. If a projection entry includes an action that the reader cannot evaluate,\n the reader MUST fail rather than ignore the transform.\n\n5. An identity transform is equivalent to projecting the column directly.\n8. The data type of the projected column MUST match the data type defined for the transform result.\n', ) required_row_filter: Optional[Expression] = Field( None, diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index 03c9880b68db..e2815bad2fab 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3666,7 +3666,7 @@ components: ReadRestrictions: type: object description: > - Read restrictions for a table, including projections and row filter expressions, according to the current schema. + Read restrictions for a table, including column projections and row filter expressions, according to the current schema. A client MUST enforce the restrictions defined in this object when reading data from the table. @@ -3676,26 +3676,23 @@ components: MUST NOT be applied beyond the entity identified by the Authentication header (or other applicable authentication mechanism). properties: - required-projections: + required-column-projections: description: > A list of projections that MUST be applied prior to any query-specified projections. - If the required-projection property is absent, no mandatory projection applies, + If the required-colum-projections property is absent, no mandatory projection applies, and a reader MAY project any subset of columns of the table, including all columns. - 1. A reader MUST project only columns listed in the required-projection. + 1. A reader MUST project only columns listed in the required-colum-projections. - If a listed column has a transform, the reader MUST apply it and replace all references to the underlying column with the transformed value (for example, truncate[4](cc) MUST be projected as truncate[4](cc) AS cc, and all references to cc during query evaluation post applying required-row-filter MUST resolve to this alias). - - Columns not listed in the required-projection MUST NOT be read. + - Columns not listed in the required-colum-projections MUST NOT be read. - 2. A column MUST appear at most once in the required-projection. + 2. A column MUST appear at most once in the required-colum-projections. - 3. Multiple transformed versions of the same column (e.g., truncate[5](col) - and truncate[3](col) MUST NOT appear in the required-projection. - - 4. If a projection entry includes an action that the reader cannot evaluate, + 3. If a projection entry includes an action that the reader cannot evaluate, the reader MUST fail rather than ignore the transform. 5. An identity transform is equivalent to projecting the column directly. @@ -3727,13 +3724,13 @@ components: type: object description: Defines a projection for a column. properties: - source-id: - type: int + field-id: + type: integer description: field id of the column being projected. action: $ref: '#/components/schemas/Action' required: - - source-id + - field-id - action Action: From 996709a26f9722095faa7cab1d8bc01ddd71f1d8 Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Mon, 10 Nov 2025 09:16:55 -0800 Subject: [PATCH 13/45] Address review feedback --- open-api/rest-catalog-open-api.py | 105 ++++++++++++++++------------ open-api/rest-catalog-open-api.yaml | 81 +++++++-------------- 2 files changed, 85 insertions(+), 101 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index 49487f3a40d4..1335e852b451 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -590,38 +590,22 @@ class StorageCredential(BaseModel): class MaskHashSha256(BaseModel): - """ - Mask the data of the column by apply SHA256 hash algorithm. Engines are free to use their own implementation of SHA256. - """ - - type: Literal['mask_hash_sha256'] - + __root__: Any = Field( + ..., + description='Mask the data of the column by applying SHA-256. \nThe input must be UTF-8 encoded bytes of the column value. \nThe SHA-256 digest is represented as a lowercase hexadecimal string. \nEngines must follow this procedure to ensure consistency:\n1. Convert the column value to a UTF-8 byte array.\n2. Apply the SHA-256 algorithm as specified in NIST FIPS 180-4.\n3. Convert the resulting 32-byte digest to a 64-character lowercase hexadecimal string.\n', + ) -class MaskReplaceWithNull(BaseModel): - """ - Masks data by replacing it with a NULL value. - """ - type: Literal['mask_replace_with_null'] +class ReplaceWithNull(BaseModel): + __root__: Any = Field( + ..., description='Masks data by replacing it with a NULL value.' + ) class MaskAlphanumeric(BaseModel): - """ - mask all alphabetic characters with 'x' and numeric characters with 'n' - """ - - type: Literal['mask_alphanumeric'] - - -class ApplyTransform(BaseModel): - """ - Applies the transform - """ - - type: Literal['apply_transform'] - transform: Transform - source_ids: List[int] = Field( - ..., alias='source-ids', description='args of transform' + __root__: Any = Field( + ..., + description="mask all alphabetic characters with 'x' and numeric characters with 'n'", ) @@ -1188,15 +1172,6 @@ class ViewRequirement(RootModel[AssertViewUUID]): root: AssertViewUUID = Field(..., discriminator='type') -class Action(BaseModel): - __root__: Union[ - MaskHashSha256, MaskReplaceWithNull, MaskAlphanumeric, ApplyTransform - ] = Field( - ..., - description='Defines the specific action to be executed for computing the projection.', - ) - - class FailedPlanningResult(IcebergErrorResponse): """ Failed server-side planning result @@ -1331,15 +1306,12 @@ class FunctionDefinitionVersion(BaseModel): ) -class Projection(BaseModel): +class ApplyTransform(BaseModel): """ - Defines a projection for a column. + Replace the field with the result of a transform expression. Produce the original field name with the transformed values. """ - field_id: int = Field( - ..., alias='field-id', description='field id of the column being projected.' - ) - action: Action + term: Optional[Term] = None class UnaryExpression(BaseModel): @@ -1436,6 +1408,47 @@ class SetExpression(BaseModel): values: list[PrimitiveTypeValue] +class ResidualFilter6(SetExpression, ResidualFilter1): + """ + An optional filter to be applied to rows in this file scan task. + If the residual is not present, the client must produce the residual or use the original filter. + """ + + +class ResidualFilter7(LiteralExpression, ResidualFilter1): + """ + An optional filter to be applied to rows in this file scan task. + If the residual is not present, the client must produce the residual or use the original filter. + """ + + +class ResidualFilter8(UnaryExpression, ResidualFilter1): + """ + An optional filter to be applied to rows in this file scan task. + If the residual is not present, the client must produce the residual or use the original filter. + """ + + +class Action(BaseModel): + __root__: Union[ + MaskHashSha256, ReplaceWithNull, MaskAlphanumeric, ApplyTransform + ] = Field( + ..., + description='Defines the specific action to be executed for computing the projection.', + ) + + +class Projection(BaseModel): + """ + Defines a projection for a column. + """ + + field_id: int = Field( + ..., alias='field-id', description='field id of the column being projected.' + ) + action: Action + + class StructField(BaseModel): id: int name: str @@ -1584,21 +1597,21 @@ class AddSchemaUpdate(BaseUpdate): class ReadRestrictions(BaseModel): """ - Read restrictions for a table, including column projections and row filter expressions, according to the current schema. + Read restrictions for a table, including column projections and row filter expressions. A client MUST enforce the restrictions defined in this object when reading data from the table. - These restrictions apply only to the authenticated principal, user, or account associated with the client. They MUST NOT be interpreted as global policy and MUST NOT be applied beyond the entity identified by the Authentication header (or other applicable authentication mechanism). + These restrictions apply only to the authenticated principal, user, or account associated with the request. They MUST NOT be interpreted as global policy and MUST NOT be applied beyond the entity identified by the Authentication header (or other applicable authentication mechanism). """ required_column_projections: Optional[List[Projection]] = Field( None, alias='required-column-projections', - description='A list of projections that MUST be applied prior to any query-specified projections. If the required-colum-projections property is absent, no mandatory projection applies, and a reader MAY project any subset of columns of the table, including all columns.\n1. A reader MUST project only columns listed in the required-colum-projections.\n - If a listed column has a transform, the reader MUST apply it and replace\n all references to the underlying column with the transformed value\n (for example, truncate[4](cc) MUST be projected as truncate[4](cc) AS cc,\n and all references to cc during query evaluation post applying required-row-filter MUST resolve to this alias).\n - Columns not listed in the required-colum-projections MUST NOT be read.\n\n2. A column MUST appear at most once in the required-colum-projections.\n3. If a projection entry includes an action that the reader cannot evaluate,\n the reader MUST fail rather than ignore the transform.\n\n5. An identity transform is equivalent to projecting the column directly.\n8. The data type of the projected column MUST match the data type defined for the transform result.\n', + description="A list of projections that MUST be applied prior to any query-specified projections. If this property is absent, no mandatory projection applies, and a reader MAY project any subset of columns of the table, including all columns.\n1. A reader MUST project only columns listed in the required-column-projections.\n - If a listed column has a transform, the reader MUST apply it and replace\n all references to the underlying column with the transformed value\n (for example, truncate[4](cc) MUST be projected as truncate[4](cc) AS cc,\n and all references to cc during query evaluation post applying required-row-filter MUST resolve to this alias).\n - Columns not listed in the required-column-projections MUST NOT be read.\n\n2. A column MUST appear at most once in the required-column-projections.\n3. If a projected column's corresponding entry includes an action that the reader cannot evaluate,\n the reader MUST fail rather than ignore the transform.\n\n4. An identity transform is equivalent to projecting the column directly.\n5. The data type of the projected column MUST match the data type defined for the transform result.\n", ) required_row_filter: Optional[Expression] = Field( None, alias='required-row-filter', - description="An expression that filters rows in the table.\n1. A reader MUST discard any row for which the filter evaluates to false or null, and\n no information derived from discarded rows MAY be included in the query result.\n\n2. Row filters MUST be evaluated against the original, untransformed column values.\n Required projections MUST be applied only after row filters are applied.\n\n3. If the catalog supports multiple row access filters for the table, it is\n the catalog's responsibility to combine them using the appropriate logic (e.g., AND, OR).\n\n4. If a client cannot interpret or evaluate a provided filter expression, it MUST fail.\n5. If the required-row-filter property is absent or empty, no mandatory filtering is imposed.\n", + description='An expression that filters rows in the table that the authenticated principal does not have access to.\n1. A reader MUST discard any row for which the filter evaluates to false or null, and\n no information derived from discarded rows MAY be included in the query result.\n\n2. Row filters MUST be evaluated against the original, untransformed column values.\n Required projections MUST be applied only after row filters are applied.\n\n3. If a client cannot interpret or evaluate a provided filter expression, it MUST fail.\n4. If this property is absent, null, or always true then no mandatory filtering is required.\n', ) diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index e2815bad2fab..1a0c4c631f55 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3666,13 +3666,13 @@ components: ReadRestrictions: type: object description: > - Read restrictions for a table, including column projections and row filter expressions, according to the current schema. + Read restrictions for a table, including column projections and row filter expressions. A client MUST enforce the restrictions defined in this object when reading data from the table. These restrictions apply only to the authenticated principal, user, or account - associated with the client. They MUST NOT be interpreted as global policy and + associated with the request. They MUST NOT be interpreted as global policy and MUST NOT be applied beyond the entity identified by the Authentication header (or other applicable authentication mechanism). properties: @@ -3680,31 +3680,31 @@ components: description: > A list of projections that MUST be applied prior to any query-specified projections. - If the required-colum-projections property is absent, no mandatory projection applies, + If this property is absent, no mandatory projection applies, and a reader MAY project any subset of columns of the table, including all columns. - 1. A reader MUST project only columns listed in the required-colum-projections. + 1. A reader MUST project only columns listed in the required-column-projections. - If a listed column has a transform, the reader MUST apply it and replace all references to the underlying column with the transformed value (for example, truncate[4](cc) MUST be projected as truncate[4](cc) AS cc, and all references to cc during query evaluation post applying required-row-filter MUST resolve to this alias). - - Columns not listed in the required-colum-projections MUST NOT be read. + - Columns not listed in the required-column-projections MUST NOT be read. - 2. A column MUST appear at most once in the required-colum-projections. + 2. A column MUST appear at most once in the required-column-projections. - 3. If a projection entry includes an action that the reader cannot evaluate, + 3. If a projected column's corresponding entry includes an action that the reader cannot evaluate, the reader MUST fail rather than ignore the transform. - 5. An identity transform is equivalent to projecting the column directly. + 4. An identity transform is equivalent to projecting the column directly. - 8. The data type of the projected column MUST match the data type defined for the transform result. + 5. The data type of the projected column MUST match the data type defined for the transform result. type: array items: $ref: '#/components/schemas/Projection' required-row-filter: description: > - An expression that filters rows in the table. + An expression that filters rows in the table that the authenticated principal does not have access to. 1. A reader MUST discard any row for which the filter evaluates to false or null, and no information derived from discarded rows MAY be included in the query result. @@ -3712,12 +3712,9 @@ components: 2. Row filters MUST be evaluated against the original, untransformed column values. Required projections MUST be applied only after row filters are applied. - 3. If the catalog supports multiple row access filters for the table, it is - the catalog's responsibility to combine them using the appropriate logic (e.g., AND, OR). + 3. If a client cannot interpret or evaluate a provided filter expression, it MUST fail. - 4. If a client cannot interpret or evaluate a provided filter expression, it MUST fail. - - 5. If the required-row-filter property is absent or empty, no mandatory filtering is imposed. + 4. If this property is absent, null, or always true then no mandatory filtering is required. $ref: '#/components/schemas/Expression' Projection: @@ -3737,58 +3734,32 @@ components: description: Defines the specific action to be executed for computing the projection. oneOf: - $ref: '#/components/schemas/MaskHashSha256' - - $ref: '#/components/schemas/MaskReplaceWithNull' + - $ref: '#/components/schemas/ReplaceWithNull' - $ref: '#/components/schemas/MaskAlphanumeric' - $ref: '#/components/schemas/ApplyTransform' MaskHashSha256: - type: object - description: Mask the data of the column by apply SHA256 hash algorithm. Engines are free to use their own implementation of SHA256. - properties: - type: - type: string - enum: [ mask_hash_sha256 ] - required: - - type - - MaskReplaceWithNull: - type: object + description: | + Mask the data of the column by applying SHA-256. + The input must be UTF-8 encoded bytes of the column value. + The SHA-256 digest is represented as a lowercase hexadecimal string. + Engines must follow this procedure to ensure consistency: + 1. Convert the column value to a UTF-8 byte array. + 2. Apply the SHA-256 algorithm as specified in NIST FIPS 180-4. + 3. Convert the resulting 32-byte digest to a 64-character lowercase hexadecimal string. + + ReplaceWithNull: description: Masks data by replacing it with a NULL value. - properties: - type: - type: string - enum: [ mask_replace_with_null ] - required: - - type MaskAlphanumeric: - type: object description: mask all alphabetic characters with 'x' and numeric characters with 'n' - properties: - type: - type: string - enum: [ mask_alphanumeric ] - required: - - type ApplyTransform: type: object - description: Applies the transform + description: Replace the field with the result of a transform expression. Produce the original field name with the transformed values. properties: - type: - type: string - enum: [ apply_transform ] - transform: - $ref: '#/components/schemas/Transform' - source-ids: - description: args of transform - type: array - items: - type: integer - required: - - type - - transform - - source-ids + term: + $ref: '#/components/schemas/Term' LoadCredentialsResponse: type: object From 0a13b5aeffa0303ad3a98bba06c9b5fd98a5560d Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Sun, 14 Dec 2025 09:26:23 -0800 Subject: [PATCH 14/45] pass lint --- open-api/rest-catalog-open-api.py | 91 ++++++++++++++--------------- open-api/rest-catalog-open-api.yaml | 20 +++---- 2 files changed, 55 insertions(+), 56 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index 1335e852b451..266ea5713c5c 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -18,7 +18,7 @@ from __future__ import annotations from datetime import date, timedelta -from typing import Dict, Literal +from typing import Any, Dict, List, Literal, Optional, Union from uuid import UUID from pydantic import Base64Str, BaseModel, ConfigDict, Field, RootModel @@ -269,13 +269,10 @@ class Summary(BaseModel): model_config = ConfigDict( extra='allow', ) + __pydantic_extra__: dict[str, str] operation: Literal['append', 'replace', 'overwrite', 'delete'] -Summary.__annotations__['__pydantic_extra__'] = Dict[str, str] -Summary.model_rebuild(force=True) - - class Snapshot(BaseModel): snapshot_id: int = Field(..., alias='snapshot-id') parent_snapshot_id: int | None = Field(None, alias='parent-snapshot-id') @@ -368,17 +365,17 @@ class AssignUUIDUpdate(BaseUpdate): Assigning a UUID to a table/view should only be done when creating the table/view. It is not safe to re-assign the UUID if a table/view already has a UUID assigned """ - action: Literal['assign-uuid'] + action: Literal['assign-uuid'] = 'assign-uuid' uuid: str class UpgradeFormatVersionUpdate(BaseUpdate): - action: Literal['upgrade-format-version'] + action: Literal['upgrade-format-version'] = 'upgrade-format-version' format_version: int = Field(..., alias='format-version') class SetCurrentSchemaUpdate(BaseUpdate): - action: Literal['set-current-schema'] + action: Literal['set-current-schema'] = 'set-current-schema' schema_id: int = Field( ..., alias='schema-id', @@ -387,12 +384,12 @@ class SetCurrentSchemaUpdate(BaseUpdate): class AddPartitionSpecUpdate(BaseUpdate): - action: Literal['add-spec'] + action: Literal['add-spec'] = 'add-spec' spec: PartitionSpec class SetDefaultSpecUpdate(BaseUpdate): - action: Literal['set-default-spec'] + action: Literal['set-default-spec'] = 'set-default-spec' spec_id: int = Field( ..., alias='spec-id', @@ -401,12 +398,12 @@ class SetDefaultSpecUpdate(BaseUpdate): class AddSortOrderUpdate(BaseUpdate): - action: Literal['add-sort-order'] + action: Literal['add-sort-order'] = 'add-sort-order' sort_order: SortOrder = Field(..., alias='sort-order') class SetDefaultSortOrderUpdate(BaseUpdate): - action: Literal['set-default-sort-order'] + action: Literal['set-default-sort-order'] = 'set-default-sort-order' sort_order_id: int = Field( ..., alias='sort-order-id', @@ -415,47 +412,47 @@ class SetDefaultSortOrderUpdate(BaseUpdate): class AddSnapshotUpdate(BaseUpdate): - action: Literal['add-snapshot'] + action: Literal['add-snapshot'] = 'add-snapshot' snapshot: Snapshot class SetSnapshotRefUpdate(BaseUpdate, SnapshotReference): - action: Literal['set-snapshot-ref'] + action: Literal['set-snapshot-ref'] = 'set-snapshot-ref' ref_name: str = Field(..., alias='ref-name') class RemoveSnapshotsUpdate(BaseUpdate): - action: Literal['remove-snapshots'] + action: Literal['remove-snapshots'] = 'remove-snapshots' snapshot_ids: list[int] = Field(..., alias='snapshot-ids') class RemoveSnapshotRefUpdate(BaseUpdate): - action: Literal['remove-snapshot-ref'] + action: Literal['remove-snapshot-ref'] = 'remove-snapshot-ref' ref_name: str = Field(..., alias='ref-name') class SetLocationUpdate(BaseUpdate): - action: Literal['set-location'] + action: Literal['set-location'] = 'set-location' location: str class SetPropertiesUpdate(BaseUpdate): - action: Literal['set-properties'] + action: Literal['set-properties'] = 'set-properties' updates: dict[str, str] class RemovePropertiesUpdate(BaseUpdate): - action: Literal['remove-properties'] + action: Literal['remove-properties'] = 'remove-properties' removals: list[str] class AddViewVersionUpdate(BaseUpdate): - action: Literal['add-view-version'] + action: Literal['add-view-version'] = 'add-view-version' view_version: ViewVersion = Field(..., alias='view-version') class SetCurrentViewVersionUpdate(BaseUpdate): - action: Literal['set-current-view-version'] + action: Literal['set-current-view-version'] = 'set-current-view-version' view_version_id: int = Field( ..., alias='view-version-id', @@ -464,32 +461,32 @@ class SetCurrentViewVersionUpdate(BaseUpdate): class RemoveStatisticsUpdate(BaseUpdate): - action: Literal['remove-statistics'] + action: Literal['remove-statistics'] = 'remove-statistics' snapshot_id: int = Field(..., alias='snapshot-id') class RemovePartitionStatisticsUpdate(BaseUpdate): - action: Literal['remove-partition-statistics'] + action: Literal['remove-partition-statistics'] = 'remove-partition-statistics' snapshot_id: int = Field(..., alias='snapshot-id') class RemovePartitionSpecsUpdate(BaseUpdate): - action: Literal['remove-partition-specs'] + action: Literal['remove-partition-specs'] = 'remove-partition-specs' spec_ids: list[int] = Field(..., alias='spec-ids') class RemoveSchemasUpdate(BaseUpdate): - action: Literal['remove-schemas'] + action: Literal['remove-schemas'] = 'remove-schemas' schema_ids: list[int] = Field(..., alias='schema-ids') class AddEncryptionKeyUpdate(BaseUpdate): - action: Literal['add-encryption-key'] + action: Literal['add-encryption-key'] = 'add-encryption-key' encryption_key: EncryptedKey = Field(..., alias='encryption-key') class RemoveEncryptionKeyUpdate(BaseUpdate): - action: Literal['remove-encryption-key'] + action: Literal['remove-encryption-key'] = 'remove-encryption-key' key_id: str = Field(..., alias='key-id') @@ -522,7 +519,7 @@ class AssertRefSnapshotId(TableRequirement): """ - type: Literal['assert-ref-snapshot-id'] + type: Literal['assert-ref-snapshot-id'] = 'assert-ref-snapshot-id' ref: str snapshot_id: int = Field(..., alias='snapshot-id') @@ -532,7 +529,7 @@ class AssertLastAssignedFieldId(TableRequirement): The table's last assigned column id must match the requirement's `last-assigned-field-id` """ - type: Literal['assert-last-assigned-field-id'] + type: Literal['assert-last-assigned-field-id'] = 'assert-last-assigned-field-id' last_assigned_field_id: int = Field(..., alias='last-assigned-field-id') @@ -541,7 +538,7 @@ class AssertCurrentSchemaId(TableRequirement): The table's current schema id must match the requirement's `current-schema-id` """ - type: Literal['assert-current-schema-id'] + type: Literal['assert-current-schema-id'] = 'assert-current-schema-id' current_schema_id: int = Field(..., alias='current-schema-id') @@ -550,7 +547,9 @@ class AssertLastAssignedPartitionId(TableRequirement): The table's last assigned partition id must match the requirement's `last-assigned-partition-id` """ - type: Literal['assert-last-assigned-partition-id'] + type: Literal['assert-last-assigned-partition-id'] = ( + 'assert-last-assigned-partition-id' + ) last_assigned_partition_id: int = Field(..., alias='last-assigned-partition-id') @@ -559,7 +558,7 @@ class AssertDefaultSpecId(TableRequirement): The table's default spec id must match the requirement's `default-spec-id` """ - type: Literal['assert-default-spec-id'] + type: Literal['assert-default-spec-id'] = 'assert-default-spec-id' default_spec_id: int = Field(..., alias='default-spec-id') @@ -568,7 +567,7 @@ class AssertDefaultSortOrderId(TableRequirement): The table's default sort order id must match the requirement's `default-sort-order-id` """ - type: Literal['assert-default-sort-order-id'] + type: Literal['assert-default-sort-order-id'] = 'assert-default-sort-order-id' default_sort_order_id: int = Field(..., alias='default-sort-order-id') @@ -592,7 +591,7 @@ class StorageCredential(BaseModel): class MaskHashSha256(BaseModel): __root__: Any = Field( ..., - description='Mask the data of the column by applying SHA-256. \nThe input must be UTF-8 encoded bytes of the column value. \nThe SHA-256 digest is represented as a lowercase hexadecimal string. \nEngines must follow this procedure to ensure consistency:\n1. Convert the column value to a UTF-8 byte array.\n2. Apply the SHA-256 algorithm as specified in NIST FIPS 180-4.\n3. Convert the resulting 32-byte digest to a 64-character lowercase hexadecimal string.\n', + description='Mask the data of the column by applying SHA-256.\nThe input must be UTF-8 encoded bytes of the column value.\nThe SHA-256 digest is represented as a lowercase hexadecimal string.\nEngines must follow this procedure to ensure consistency:\n1. Convert the column value to a UTF-8 byte array.\n2. Apply the SHA-256 algorithm as specified in NIST FIPS 180-4.\n3. Convert the resulting 32-byte digest to a 64-character lowercase hexadecimal string.\n', ) @@ -1162,7 +1161,7 @@ class TransformTerm(BaseModel): class SetPartitionStatisticsUpdate(BaseUpdate): - action: Literal['set-partition-statistics'] + action: Literal['set-partition-statistics'] = 'set-partition-statistics' partition_statistics: PartitionStatisticsFile = Field( ..., alias='partition-statistics' ) @@ -1272,7 +1271,7 @@ class Term(RootModel[Reference | TransformTerm]): class SetStatisticsUpdate(BaseUpdate): - action: Literal['set-statistics'] + action: Literal['set-statistics'] = 'set-statistics' snapshot_id: int | None = Field( None, alias='snapshot-id', @@ -1408,6 +1407,15 @@ class SetExpression(BaseModel): values: list[PrimitiveTypeValue] +class Action(BaseModel): + __root__: Union[ + MaskHashSha256, ReplaceWithNull, MaskAlphanumeric, ApplyTransform + ] = Field( + ..., + description='Defines the specific action to be executed for computing the projection.', + ) + + class ResidualFilter6(SetExpression, ResidualFilter1): """ An optional filter to be applied to rows in this file scan task. @@ -1429,15 +1437,6 @@ class ResidualFilter8(UnaryExpression, ResidualFilter1): """ -class Action(BaseModel): - __root__: Union[ - MaskHashSha256, ReplaceWithNull, MaskAlphanumeric, ApplyTransform - ] = Field( - ..., - description='Defines the specific action to be executed for computing the projection.', - ) - - class Projection(BaseModel): """ Defines a projection for a column. @@ -1585,7 +1584,7 @@ class ViewMetadata(BaseModel): class AddSchemaUpdate(BaseUpdate): - action: Literal['add-schema'] + action: Literal['add-schema'] = 'add-schema' schema_: Schema = Field(..., alias='schema') last_column_id: int | None = Field( None, diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index 1a0c4c631f55..4e35bf256bb7 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3691,12 +3691,12 @@ components: - Columns not listed in the required-column-projections MUST NOT be read. 2. A column MUST appear at most once in the required-column-projections. - + 3. If a projected column's corresponding entry includes an action that the reader cannot evaluate, the reader MUST fail rather than ignore the transform. - + 4. An identity transform is equivalent to projecting the column directly. - + 5. The data type of the projected column MUST match the data type defined for the transform result. type: array @@ -3705,15 +3705,15 @@ components: required-row-filter: description: > An expression that filters rows in the table that the authenticated principal does not have access to. - + 1. A reader MUST discard any row for which the filter evaluates to false or null, and no information derived from discarded rows MAY be included in the query result. - + 2. Row filters MUST be evaluated against the original, untransformed column values. Required projections MUST be applied only after row filters are applied. - + 3. If a client cannot interpret or evaluate a provided filter expression, it MUST fail. - + 4. If this property is absent, null, or always true then no mandatory filtering is required. $ref: '#/components/schemas/Expression' @@ -3740,9 +3740,9 @@ components: MaskHashSha256: description: | - Mask the data of the column by applying SHA-256. - The input must be UTF-8 encoded bytes of the column value. - The SHA-256 digest is represented as a lowercase hexadecimal string. + Mask the data of the column by applying SHA-256. + The input must be UTF-8 encoded bytes of the column value. + The SHA-256 digest is represented as a lowercase hexadecimal string. Engines must follow this procedure to ensure consistency: 1. Convert the column value to a UTF-8 byte array. 2. Apply the SHA-256 algorithm as specified in NIST FIPS 180-4. From 794f52eb1e2b7a69477326b9ce250e59272dff7a Mon Sep 17 00:00:00 2001 From: Prashant Kumar Singh Date: Sun, 11 Jan 2026 05:15:42 +0000 Subject: [PATCH 15/45] Address review feedback from steven --- open-api/rest-catalog-open-api.py | 48 ++++++++--------------------- open-api/rest-catalog-open-api.yaml | 5 +-- 2 files changed, 16 insertions(+), 37 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index 266ea5713c5c..f014f9dd087f 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -18,7 +18,7 @@ from __future__ import annotations from datetime import date, timedelta -from typing import Any, Dict, List, Literal, Optional, Union +from typing import Any, Literal from uuid import UUID from pydantic import Base64Str, BaseModel, ConfigDict, Field, RootModel @@ -1310,7 +1310,7 @@ class ApplyTransform(BaseModel): Replace the field with the result of a transform expression. Produce the original field name with the transformed values. """ - term: Optional[Term] = None + term: Term | None = None class UnaryExpression(BaseModel): @@ -1408,44 +1408,24 @@ class SetExpression(BaseModel): class Action(BaseModel): - __root__: Union[ - MaskHashSha256, ReplaceWithNull, MaskAlphanumeric, ApplyTransform - ] = Field( - ..., - description='Defines the specific action to be executed for computing the projection.', + __root__: MaskHashSha256 | ReplaceWithNull | MaskAlphanumeric | ApplyTransform = ( + Field( + ..., + description='Defines the specific action to be executed for computing the projection.', + ) ) -class ResidualFilter6(SetExpression, ResidualFilter1): - """ - An optional filter to be applied to rows in this file scan task. - If the residual is not present, the client must produce the residual or use the original filter. - """ - - -class ResidualFilter7(LiteralExpression, ResidualFilter1): - """ - An optional filter to be applied to rows in this file scan task. - If the residual is not present, the client must produce the residual or use the original filter. - """ - - -class ResidualFilter8(UnaryExpression, ResidualFilter1): - """ - An optional filter to be applied to rows in this file scan task. - If the residual is not present, the client must produce the residual or use the original filter. - """ - - class Projection(BaseModel): """ - Defines a projection for a column. + Defines a projection for a column. If action is not specified, the column is projected as-is. + """ field_id: int = Field( ..., alias='field-id', description='field id of the column being projected.' ) - action: Action + action: Action | None = None class StructField(BaseModel): @@ -1602,12 +1582,12 @@ class ReadRestrictions(BaseModel): """ - required_column_projections: Optional[List[Projection]] = Field( + required_column_projections: list[Projection] | None = Field( None, alias='required-column-projections', description="A list of projections that MUST be applied prior to any query-specified projections. If this property is absent, no mandatory projection applies, and a reader MAY project any subset of columns of the table, including all columns.\n1. A reader MUST project only columns listed in the required-column-projections.\n - If a listed column has a transform, the reader MUST apply it and replace\n all references to the underlying column with the transformed value\n (for example, truncate[4](cc) MUST be projected as truncate[4](cc) AS cc,\n and all references to cc during query evaluation post applying required-row-filter MUST resolve to this alias).\n - Columns not listed in the required-column-projections MUST NOT be read.\n\n2. A column MUST appear at most once in the required-column-projections.\n3. If a projected column's corresponding entry includes an action that the reader cannot evaluate,\n the reader MUST fail rather than ignore the transform.\n\n4. An identity transform is equivalent to projecting the column directly.\n5. The data type of the projected column MUST match the data type defined for the transform result.\n", ) - required_row_filter: Optional[Expression] = Field( + required_row_filter: Expression | None = Field( None, alias='required-row-filter', description='An expression that filters rows in the table that the authenticated principal does not have access to.\n1. A reader MUST discard any row for which the filter evaluates to false or null, and\n no information derived from discarded rows MAY be included in the query result.\n\n2. Row filters MUST be evaluated against the original, untransformed column values.\n Required projections MUST be applied only after row filters are applied.\n\n3. If a client cannot interpret or evaluate a provided filter expression, it MUST fail.\n4. If this property is absent, null, or always true then no mandatory filtering is required.\n', @@ -1669,9 +1649,7 @@ class LoadTableResult(BaseModel): storage_credentials: list[StorageCredential] | None = Field( None, alias='storage-credentials' ) - read_restrictions: Optional[ReadRestrictions] = Field( - None, alias='read-restrictions' - ) + read_restrictions: ReadRestrictions | None = Field(None, alias='read-restrictions') class ScanTasks(BaseModel): diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index 4e35bf256bb7..24b5235baac6 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3719,7 +3719,9 @@ components: Projection: type: object - description: Defines a projection for a column. + description: > + Defines a projection for a column. + If action is not specified, the column is projected as-is. properties: field-id: type: integer @@ -3728,7 +3730,6 @@ components: $ref: '#/components/schemas/Action' required: - field-id - - action Action: description: Defines the specific action to be executed for computing the projection. From b7e61f6f8c7322f81c6b2eafcd6bd0963d38da1d Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Tue, 17 Feb 2026 07:09:18 -0800 Subject: [PATCH 16/45] Address feedbacks per sync 02/03/2026 --- open-api/rest-catalog-open-api.py | 2 +- open-api/rest-catalog-open-api.yaml | 43 +++++++++++++++++++---------- 2 files changed, 29 insertions(+), 16 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index f014f9dd087f..8e43db502edd 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -1585,7 +1585,7 @@ class ReadRestrictions(BaseModel): required_column_projections: list[Projection] | None = Field( None, alias='required-column-projections', - description="A list of projections that MUST be applied prior to any query-specified projections. If this property is absent, no mandatory projection applies, and a reader MAY project any subset of columns of the table, including all columns.\n1. A reader MUST project only columns listed in the required-column-projections.\n - If a listed column has a transform, the reader MUST apply it and replace\n all references to the underlying column with the transformed value\n (for example, truncate[4](cc) MUST be projected as truncate[4](cc) AS cc,\n and all references to cc during query evaluation post applying required-row-filter MUST resolve to this alias).\n - Columns not listed in the required-column-projections MUST NOT be read.\n\n2. A column MUST appear at most once in the required-column-projections.\n3. If a projected column's corresponding entry includes an action that the reader cannot evaluate,\n the reader MUST fail rather than ignore the transform.\n\n4. An identity transform is equivalent to projecting the column directly.\n5. The data type of the projected column MUST match the data type defined for the transform result.\n", + description="A list of columns that require specific projections or transforms to be applied.\nIf this property is absent, a reader MAY access all columns of the table as-is without any mandatory transformations.\nIf this property is present, each listed column MUST have its specified projection or action applied. Columns not listed in required-column-projections MAY be read as-is without any restrictions.\nWhen this list is present:\n1. For each column listed in required-column-projections, the reader MUST apply\n the specified action or transform.\n\n2. If a listed column has an action, the reader MUST apply it and replace\n all references to the underlying column with the transformed value.\n For example, if the action specifies truncate[4](cc), the reader MUST project it\n as truncate[4](cc) AS cc, and all references to cc during query evaluation\n (including in required-row-filter) MUST resolve to this transformed alias.\n\n3. If a listed column has no action specified, the reader MUST project the column as-is\n (equivalent to an identity transform).\n\n4. Columns not listed in required-column-projections MAY be projected normally\n by the reader without any mandatory transformations.\n\n5. A column MUST appear at most once in required-column-projections.\n6. If a projected column's action cannot be evaluated by the reader,\n the reader MUST fail rather than ignore or skip the transform.\n\n7. The data type of a projected column MUST match the data type defined for\n the transform result or the original column type if no transform is specified.\n", ) required_row_filter: Expression | None = Field( None, diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index 24b5235baac6..f372cfde157e 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3678,26 +3678,39 @@ components: properties: required-column-projections: description: > - A list of projections that MUST be applied prior to any query-specified - projections. - If this property is absent, no mandatory projection applies, - and a reader MAY project any subset of columns of the table, including all columns. + A list of columns that require specific projections or transforms to be applied. - 1. A reader MUST project only columns listed in the required-column-projections. - - If a listed column has a transform, the reader MUST apply it and replace - all references to the underlying column with the transformed value - (for example, truncate[4](cc) MUST be projected as truncate[4](cc) AS cc, - and all references to cc during query evaluation post applying required-row-filter MUST resolve to this alias). - - Columns not listed in the required-column-projections MUST NOT be read. + If this property is absent, a reader MAY access all columns of the table as-is + without any mandatory transformations. - 2. A column MUST appear at most once in the required-column-projections. + If this property is present, each listed column MUST have its specified + projection or action applied. Columns not listed in required-column-projections + MAY be read as-is without any restrictions. - 3. If a projected column's corresponding entry includes an action that the reader cannot evaluate, - the reader MUST fail rather than ignore the transform. + When this list is present: - 4. An identity transform is equivalent to projecting the column directly. + 1. For each column listed in required-column-projections, the reader MUST apply + the specified action or transform. - 5. The data type of the projected column MUST match the data type defined for the transform result. + 2. If a listed column has an action, the reader MUST apply it and replace + all references to the underlying column with the transformed value. + For example, if the action specifies truncate[4](cc), the reader MUST project it + as truncate[4](cc) AS cc, and all references to cc during query evaluation + (including in required-row-filter) MUST resolve to this transformed alias. + + 3. If a listed column has no action specified, the reader MUST project the column as-is + (equivalent to an identity transform). + + 4. Columns not listed in required-column-projections MAY be projected normally + by the reader without any mandatory transformations. + + 5. A column MUST appear at most once in required-column-projections. + + 6. If a projected column's action cannot be evaluated by the reader, + the reader MUST fail rather than ignore or skip the transform. + + 7. The data type of a projected column MUST match the data type defined for + the transform result or the original column type if no transform is specified. type: array items: From 9cf12e7584a72b9038dbb727279113f543a13919 Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Sun, 8 Mar 2026 21:41:59 -0700 Subject: [PATCH 17/45] Address comment: clarify unlisted columns wording --- open-api/rest-catalog-open-api.py | 2 +- open-api/rest-catalog-open-api.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index 8e43db502edd..530a06952b00 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -1585,7 +1585,7 @@ class ReadRestrictions(BaseModel): required_column_projections: list[Projection] | None = Field( None, alias='required-column-projections', - description="A list of columns that require specific projections or transforms to be applied.\nIf this property is absent, a reader MAY access all columns of the table as-is without any mandatory transformations.\nIf this property is present, each listed column MUST have its specified projection or action applied. Columns not listed in required-column-projections MAY be read as-is without any restrictions.\nWhen this list is present:\n1. For each column listed in required-column-projections, the reader MUST apply\n the specified action or transform.\n\n2. If a listed column has an action, the reader MUST apply it and replace\n all references to the underlying column with the transformed value.\n For example, if the action specifies truncate[4](cc), the reader MUST project it\n as truncate[4](cc) AS cc, and all references to cc during query evaluation\n (including in required-row-filter) MUST resolve to this transformed alias.\n\n3. If a listed column has no action specified, the reader MUST project the column as-is\n (equivalent to an identity transform).\n\n4. Columns not listed in required-column-projections MAY be projected normally\n by the reader without any mandatory transformations.\n\n5. A column MUST appear at most once in required-column-projections.\n6. If a projected column's action cannot be evaluated by the reader,\n the reader MUST fail rather than ignore or skip the transform.\n\n7. The data type of a projected column MUST match the data type defined for\n the transform result or the original column type if no transform is specified.\n", + description="A list of columns that require specific projections or transforms to be applied.\nIf this property is absent, a reader MAY access all columns of the table as-is without any mandatory transformations.\nIf this property is present, each listed column MUST have its specified projection or action applied. Columns not listed in required-column-projections are not subject to any read restrictions.\nWhen this list is present:\n1. For each column listed in required-column-projections, the reader MUST apply\n the specified action or transform.\n\n2. If a listed column has an action, the reader MUST apply it and replace\n all references to the underlying column with the transformed value.\n For example, if the action specifies truncate[4](cc), the reader MUST project it\n as truncate[4](cc) AS cc, and all references to cc during query evaluation\n (including in required-row-filter) MUST resolve to this transformed alias.\n\n3. If a listed column has no action specified, the reader MUST project the column as-is\n (equivalent to an identity transform).\n\n4. Columns not listed in required-column-projections MAY be projected normally\n by the reader without any mandatory transformations.\n\n5. A column MUST appear at most once in required-column-projections.\n6. If a projected column's action cannot be evaluated by the reader,\n the reader MUST fail rather than ignore or skip the transform.\n\n7. The data type of a projected column MUST match the data type defined for\n the transform result or the original column type if no transform is specified.\n", ) required_row_filter: Expression | None = Field( None, diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index f372cfde157e..f211f8846702 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3685,7 +3685,7 @@ components: If this property is present, each listed column MUST have its specified projection or action applied. Columns not listed in required-column-projections - MAY be read as-is without any restrictions. + are not subject to any read restrictions. When this list is present: From 94ae9135012d26e06f203b06d28910bb73e402b8 Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Sun, 15 Mar 2026 19:31:24 -0700 Subject: [PATCH 18/45] Add discriminator for Action and remove Projection wrapper Flatten Projection into Action with an "action" discriminator property, matching the BaseUpdate pattern. Each Action subtype now uses allOf to extend Action with a const action value, making them distinguishable in JSON. --- open-api/rest-catalog-open-api.py | 77 ++++++++++++++-------------- open-api/rest-catalog-open-api.yaml | 78 +++++++++++++++++------------ 2 files changed, 86 insertions(+), 69 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index 530a06952b00..0f7d9601e864 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -18,7 +18,7 @@ from __future__ import annotations from datetime import date, timedelta -from typing import Any, Literal +from typing import Literal from uuid import UUID from pydantic import Base64Str, BaseModel, ConfigDict, Field, RootModel @@ -588,24 +588,42 @@ class StorageCredential(BaseModel): config: dict[str, str] -class MaskHashSha256(BaseModel): - __root__: Any = Field( - ..., - description='Mask the data of the column by applying SHA-256.\nThe input must be UTF-8 encoded bytes of the column value.\nThe SHA-256 digest is represented as a lowercase hexadecimal string.\nEngines must follow this procedure to ensure consistency:\n1. Convert the column value to a UTF-8 byte array.\n2. Apply the SHA-256 algorithm as specified in NIST FIPS 180-4.\n3. Convert the resulting 32-byte digest to a 64-character lowercase hexadecimal string.\n', +class Action(BaseModel): + action: str + field_id: int = Field( + ..., alias='field-id', description='field id of the column being projected.' ) -class ReplaceWithNull(BaseModel): - __root__: Any = Field( - ..., description='Masks data by replacing it with a NULL value.' - ) +class MaskHashSha256(Action): + """ + Mask the data of the column by applying SHA-256. + The input must be UTF-8 encoded bytes of the column value. + The SHA-256 digest is represented as a lowercase hexadecimal string. + Engines must follow this procedure to ensure consistency: + 1. Convert the column value to a UTF-8 byte array. + 2. Apply the SHA-256 algorithm as specified in NIST FIPS 180-4. + 3. Convert the resulting 32-byte digest to a 64-character lowercase hexadecimal string. + """ -class MaskAlphanumeric(BaseModel): - __root__: Any = Field( - ..., - description="mask all alphabetic characters with 'x' and numeric characters with 'n'", - ) + action: Literal['mask-hash-sha256'] = Field('mask-hash-sha256', const=True) + + +class ReplaceWithNull(Action): + """ + Masks data by replacing it with a NULL value. + """ + + action: Literal['replace-with-null'] = Field('replace-with-null', const=True) + + +class MaskAlphanumeric(Action): + """ + mask all alphabetic characters with 'x' and numeric characters with 'n' + """ + + action: Literal['mask-alphanumeric'] = Field('mask-alphanumeric', const=True) class LoadCredentialsResponse(BaseModel): @@ -1305,11 +1323,12 @@ class FunctionDefinitionVersion(BaseModel): ) -class ApplyTransform(BaseModel): +class ApplyTransform(Action): """ Replace the field with the result of a transform expression. Produce the original field name with the transformed values. """ + action: Literal['apply-transform'] = Field('apply-transform', const=True) term: Term | None = None @@ -1407,27 +1426,6 @@ class SetExpression(BaseModel): values: list[PrimitiveTypeValue] -class Action(BaseModel): - __root__: MaskHashSha256 | ReplaceWithNull | MaskAlphanumeric | ApplyTransform = ( - Field( - ..., - description='Defines the specific action to be executed for computing the projection.', - ) - ) - - -class Projection(BaseModel): - """ - Defines a projection for a column. If action is not specified, the column is projected as-is. - - """ - - field_id: int = Field( - ..., alias='field-id', description='field id of the column being projected.' - ) - action: Action | None = None - - class StructField(BaseModel): id: int name: str @@ -1582,10 +1580,13 @@ class ReadRestrictions(BaseModel): """ - required_column_projections: list[Projection] | None = Field( + required_column_projections: ( + list[MaskHashSha256 | ReplaceWithNull | MaskAlphanumeric | ApplyTransform] + | None + ) = Field( None, alias='required-column-projections', - description="A list of columns that require specific projections or transforms to be applied.\nIf this property is absent, a reader MAY access all columns of the table as-is without any mandatory transformations.\nIf this property is present, each listed column MUST have its specified projection or action applied. Columns not listed in required-column-projections are not subject to any read restrictions.\nWhen this list is present:\n1. For each column listed in required-column-projections, the reader MUST apply\n the specified action or transform.\n\n2. If a listed column has an action, the reader MUST apply it and replace\n all references to the underlying column with the transformed value.\n For example, if the action specifies truncate[4](cc), the reader MUST project it\n as truncate[4](cc) AS cc, and all references to cc during query evaluation\n (including in required-row-filter) MUST resolve to this transformed alias.\n\n3. If a listed column has no action specified, the reader MUST project the column as-is\n (equivalent to an identity transform).\n\n4. Columns not listed in required-column-projections MAY be projected normally\n by the reader without any mandatory transformations.\n\n5. A column MUST appear at most once in required-column-projections.\n6. If a projected column's action cannot be evaluated by the reader,\n the reader MUST fail rather than ignore or skip the transform.\n\n7. The data type of a projected column MUST match the data type defined for\n the transform result or the original column type if no transform is specified.\n", + description="A list of columns that require specific projections or transforms to be applied.\nIf this property is absent, a reader MAY access all columns of the table as-is without any mandatory transformations.\nIf this property is present, each listed column MUST have its specified projection or action applied. Columns not listed in required-column-projections are not subject to any read restrictions.\nWhen this list is present:\n1. For each column listed in required-column-projections, the reader MUST apply\n the specified action or transform.\n\n2. The reader MUST apply the action and replace all references to the underlying\n column with the transformed value. For example, if the action specifies\n truncate[4](cc), the reader MUST project it as truncate[4](cc) AS cc, and all\n references to cc during query evaluation (including in required-row-filter)\n MUST resolve to this transformed alias.\n\n3. Columns not listed in required-column-projections MAY be projected normally\n by the reader without any mandatory transformations.\n\n4. A column MUST appear at most once in required-column-projections.\n5. If a projected column's action cannot be evaluated by the reader,\n the reader MUST fail rather than ignore or skip the transform.\n\n6. The data type of a projected column MUST match the data type defined for\n the transform result.\n", ) required_row_filter: Expression | None = Field( None, diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index f211f8846702..bb1ec2064951 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3692,29 +3692,26 @@ components: 1. For each column listed in required-column-projections, the reader MUST apply the specified action or transform. - 2. If a listed column has an action, the reader MUST apply it and replace - all references to the underlying column with the transformed value. - For example, if the action specifies truncate[4](cc), the reader MUST project it - as truncate[4](cc) AS cc, and all references to cc during query evaluation - (including in required-row-filter) MUST resolve to this transformed alias. + 2. The reader MUST apply the action and replace all references to the underlying + column with the transformed value. For example, if the action specifies + truncate[4](cc), the reader MUST project it as truncate[4](cc) AS cc, and all + references to cc during query evaluation (including in required-row-filter) + MUST resolve to this transformed alias. - 3. If a listed column has no action specified, the reader MUST project the column as-is - (equivalent to an identity transform). - - 4. Columns not listed in required-column-projections MAY be projected normally + 3. Columns not listed in required-column-projections MAY be projected normally by the reader without any mandatory transformations. - 5. A column MUST appear at most once in required-column-projections. + 4. A column MUST appear at most once in required-column-projections. - 6. If a projected column's action cannot be evaluated by the reader, + 5. If a projected column's action cannot be evaluated by the reader, the reader MUST fail rather than ignore or skip the transform. - 7. The data type of a projected column MUST match the data type defined for - the transform result or the original column type if no transform is specified. + 6. The data type of a projected column MUST match the data type defined for + the transform result. type: array items: - $ref: '#/components/schemas/Projection' + $ref: '#/components/schemas/Action' required-row-filter: description: > An expression that filters rows in the table that the authenticated principal does not have access to. @@ -3730,27 +3727,24 @@ components: 4. If this property is absent, null, or always true then no mandatory filtering is required. $ref: '#/components/schemas/Expression' - Projection: + Action: + discriminator: + propertyName: action + mapping: + mask-hash-sha256: '#/components/schemas/MaskHashSha256' + replace-with-null: '#/components/schemas/ReplaceWithNull' + mask-alphanumeric: '#/components/schemas/MaskAlphanumeric' + apply-transform: '#/components/schemas/ApplyTransform' type: object - description: > - Defines a projection for a column. - If action is not specified, the column is projected as-is. + required: + - action + - field-id properties: + action: + type: string field-id: type: integer description: field id of the column being projected. - action: - $ref: '#/components/schemas/Action' - required: - - field-id - - Action: - description: Defines the specific action to be executed for computing the projection. - oneOf: - - $ref: '#/components/schemas/MaskHashSha256' - - $ref: '#/components/schemas/ReplaceWithNull' - - $ref: '#/components/schemas/MaskAlphanumeric' - - $ref: '#/components/schemas/ApplyTransform' MaskHashSha256: description: | @@ -3761,17 +3755,39 @@ components: 1. Convert the column value to a UTF-8 byte array. 2. Apply the SHA-256 algorithm as specified in NIST FIPS 180-4. 3. Convert the resulting 32-byte digest to a 64-character lowercase hexadecimal string. + allOf: + - $ref: '#/components/schemas/Action' + properties: + action: + type: string + const: "mask-hash-sha256" ReplaceWithNull: description: Masks data by replacing it with a NULL value. + allOf: + - $ref: '#/components/schemas/Action' + properties: + action: + type: string + const: "replace-with-null" MaskAlphanumeric: description: mask all alphabetic characters with 'x' and numeric characters with 'n' + allOf: + - $ref: '#/components/schemas/Action' + properties: + action: + type: string + const: "mask-alphanumeric" ApplyTransform: - type: object description: Replace the field with the result of a transform expression. Produce the original field name with the transformed values. + allOf: + - $ref: '#/components/schemas/Action' properties: + action: + type: string + const: "apply-transform" term: $ref: '#/components/schemas/Term' From a14cdad4564109bb1f15e9c08bfac64f212a821d Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Fri, 17 Apr 2026 15:36:40 -0700 Subject: [PATCH 19/45] Model predefined masks as Actions and replace ApplyTransform with ApplyExpression - Replace 4 original actions (mask-hash-sha256, replace-with-null, mask-alphanumeric, apply-transform) with 10 predefined action types: mask-alphanum, mask-to-default, replace-with-null, show-first-4, show-last-4, truncate-to-year, truncate-to-month, sha-256-global, sha-256-query-local, apply-expression - Add discriminator mapping for all action subtypes - Add detailed descriptions for each action including type applicability, encoding rules for SHA-256 variants, and type-specific defaults - Replace Term-based ApplyTransform with Expression-based ApplyExpression - Update required-column-projections rules for action model - Regenerate Python from updated YAML spec --- open-api/rest-catalog-open-api.py | 161 ++++++++++++++++---- open-api/rest-catalog-open-api.yaml | 221 +++++++++++++++++++++++----- 2 files changed, 324 insertions(+), 58 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index 0f7d9601e864..2dc8f6b8be91 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -595,35 +595,134 @@ class Action(BaseModel): ) -class MaskHashSha256(Action): +class MaskAlphanum(Action): """ - Mask the data of the column by applying SHA-256. - The input must be UTF-8 encoded bytes of the column value. - The SHA-256 digest is represented as a lowercase hexadecimal string. - Engines must follow this procedure to ensure consistency: - 1. Convert the column value to a UTF-8 byte array. - 2. Apply the SHA-256 algorithm as specified in NIST FIPS 180-4. - 3. Convert the resulting 32-byte digest to a 64-character lowercase hexadecimal string. + Redacts the column value character by character using the following rules: + - Digits (U+0030–U+0039, 0-9) are replaced with 'n' - The following punctuation characters are kept as-is: + U+0028 '(' LEFT PARENTHESIS + U+0029 ')' RIGHT PARENTHESIS + U+002C ',' COMMA + U+002E '.' FULL STOP + U+002D '-' HYPHEN-MINUS + U+0040 '@' COMMERCIAL AT + - All other Unicode characters (including letters, whitespace, and any punctuation + not listed above) are replaced with 'x' + + For example: "prashant010696@gmail.com" → "xxxxxxxxnnnnnn@xxxxx.xxx" + Applicable to: string + + """ + + action: Literal['mask-alphanum'] = 'mask-alphanum' + + +class MaskToDefault(Action): + """ + Replaces the column value with a predefined type-specific default that conceals the original data while preserving type compatibility. Engines MUST use exactly the values listed below to ensure consistency across implementations. + Default values by type: - boolean: false - int: 999999999 - long: 999999999 - float: 0.0 - double: 0.0 - decimal(p, s): 0 (zero with s digits after the decimal point, e.g. 0.00 for decimal(p,2)) - string: "XXXXXXXX" - date: 9999-12-31 - time: 00:00:00 - timestamp: 9999-12-31T00:00:00 - timestamptz: 9999-12-31T00:00:00+00:00 - timestamp_ns: 9999-12-31T00:00:00.000000000 - timestamptz_ns: 9999-12-31T00:00:00.000000000+00:00 - uuid: 00000000-0000-0000-0000-000000000000 - fixed(n): n zero bytes - binary: empty byte sequence - variant: {"masked": true} - geometry: POINT EMPTY - geography: POINT EMPTY - list: empty list [] - map: empty map {} - struct: struct with each field set to its type-specific default (applied recursively) + Applicable to: all data types """ - action: Literal['mask-hash-sha256'] = Field('mask-hash-sha256', const=True) + action: Literal['mask-to-default'] = 'mask-to-default' class ReplaceWithNull(Action): """ - Masks data by replacing it with a NULL value. + Replaces the entire column value with NULL. + Applicable to: all nullable types + + """ + + action: Literal['replace-with-null'] = 'replace-with-null' + + +class ShowFirst4(Action): + """ + Preserves the first 4 characters of the column value and redacts the remainder using mask-alphanum rules (see MaskAlphanum for the exact character rules). Values with 4 or fewer characters are returned unchanged. + For example: "prashant010696@gmail.com" → "prasxxxxnnnnnn@xxxxx.xxx" + Applicable to: string + + """ + + action: Literal['show-first-4'] = 'show-first-4' + + +class ShowLast4(Action): + """ + Redacts all characters except the last 4 using mask-alphanum rules (see MaskAlphanum for the exact character rules). Values with 4 or fewer characters are returned unchanged. + For example: "4111-1111-1111-4444" → "nnnn-nnnn-nnnn-4444" + Applicable to: string + + """ + + action: Literal['show-last-4'] = 'show-last-4' + + +class TruncateToYear(Action): """ + Truncates the column value to year precision, setting month, day, and time components to their minimum values. The output type matches the input type. + For example: 2024-07-15 → 2024-01-01 + Applicable to: date, timestamp, timestamptz, timestamp_ns, timestamptz_ns - action: Literal['replace-with-null'] = Field('replace-with-null', const=True) + """ + action: Literal['truncate-to-year'] = 'truncate-to-year' -class MaskAlphanumeric(Action): + +class TruncateToMonth(Action): """ - mask all alphabetic characters with 'x' and numeric characters with 'n' + Truncates the column value to year and month precision, setting day and time components to their minimum values. The output type matches the input type. + For example: 2024-07-15 → 2024-07-01 + Applicable to: date, timestamp, timestamptz, timestamp_ns, timestamptz_ns + """ - action: Literal['mask-alphanumeric'] = Field('mask-alphanumeric', const=True) + action: Literal['truncate-to-month'] = 'truncate-to-month' + + +class Sha256Global(Action): + """ + Applies SHA-256 as specified in NIST FIPS 180-4. Deterministic across all queries + and engines — the same input always produces the same output. + + Input-to-bytes encoding by type: + - string: UTF-8 encoded bytes + - int: 4 bytes, little-endian + - long: 8 bytes, little-endian + - uuid: 16 bytes, big-endian (RFC 4122 byte order) + - binary: raw bytes as-is + + Output encoding by type: + - string: 64-character lowercase hexadecimal string + - int: first 4 bytes of the digest, read as a little-endian int + - long: first 8 bytes of the digest, read as a little-endian long + - uuid: all 16 bytes of the digest, interpreted as a UUID (RFC 4122 byte order) + - binary: the full 32-byte raw SHA-256 digest, output type is binary(32) + + Applicable to: string, int, long, uuid, binary + + """ + + action: Literal['sha-256-global'] = 'sha-256-global' + + +class Sha256QueryLocal(Action): + """ + Applies SHA-256 with a per-query random salt, making the output non-deterministic + across queries while remaining consistent within a single query. + + The engine MUST generate a cryptographically random salt for each query and apply it as: + SHA-256(salt_bytes || canonical_bytes) + where canonical_bytes follows the same encoding rules as sha-256-global. + + Output encoding follows the same rules as sha-256-global. + + Applicable to: string, int, long, uuid, binary + + """ + + action: Literal['sha-256-query-local'] = 'sha-256-query-local' class LoadCredentialsResponse(BaseModel): @@ -1323,15 +1422,6 @@ class FunctionDefinitionVersion(BaseModel): ) -class ApplyTransform(Action): - """ - Replace the field with the result of a transform expression. Produce the original field name with the transformed values. - """ - - action: Literal['apply-transform'] = Field('apply-transform', const=True) - term: Term | None = None - - class UnaryExpression(BaseModel): type: Literal['is-null', 'not-null', 'is-nan', 'not-nan'] = Field( ..., @@ -1581,12 +1671,23 @@ class ReadRestrictions(BaseModel): """ required_column_projections: ( - list[MaskHashSha256 | ReplaceWithNull | MaskAlphanumeric | ApplyTransform] + list[ + MaskAlphanum + | MaskToDefault + | ReplaceWithNull + | ShowFirst4 + | ShowLast4 + | TruncateToYear + | TruncateToMonth + | Sha256Global + | Sha256QueryLocal + | ApplyExpression + ] | None ) = Field( None, alias='required-column-projections', - description="A list of columns that require specific projections or transforms to be applied.\nIf this property is absent, a reader MAY access all columns of the table as-is without any mandatory transformations.\nIf this property is present, each listed column MUST have its specified projection or action applied. Columns not listed in required-column-projections are not subject to any read restrictions.\nWhen this list is present:\n1. For each column listed in required-column-projections, the reader MUST apply\n the specified action or transform.\n\n2. The reader MUST apply the action and replace all references to the underlying\n column with the transformed value. For example, if the action specifies\n truncate[4](cc), the reader MUST project it as truncate[4](cc) AS cc, and all\n references to cc during query evaluation (including in required-row-filter)\n MUST resolve to this transformed alias.\n\n3. Columns not listed in required-column-projections MAY be projected normally\n by the reader without any mandatory transformations.\n\n4. A column MUST appear at most once in required-column-projections.\n5. If a projected column's action cannot be evaluated by the reader,\n the reader MUST fail rather than ignore or skip the transform.\n\n6. The data type of a projected column MUST match the data type defined for\n the transform result.\n", + description="A list of columns that require specific actions to be applied when reading.\nIf this property is absent, a reader MAY access all columns of the table as-is without any mandatory transformations.\nIf this property is present, each listed column MUST have its specified action applied. Columns not listed in required-column-projections are not subject to any read restrictions.\nWhen this list is present:\n1. For each column listed in required-column-projections, the reader MUST apply\n the specified action before returning values for that column.\n\n2. The reader MUST replace all output references to the column with the result\n of the action, presenting the result under the original column name. For\n example, if the action for column cc is mask-alphanum, the reader MUST\n return the masked value as cc in the query output.\n\n3. Columns not listed in required-column-projections MAY be projected normally\n by the reader without any mandatory transformations.\n\n4. A column MUST appear at most once in required-column-projections.\n5. If a projected column's action cannot be evaluated by the reader,\n the reader MUST fail rather than ignore or skip the action.\n\n6. Each action defines the output type for its column. For all predefined\n actions except apply-expression, the output type matches the input column\n type. For apply-expression, the output type is determined by the expression.\n", ) required_row_filter: Expression | None = Field( None, @@ -1595,6 +1696,14 @@ class ReadRestrictions(BaseModel): ) +class ApplyExpression(Action): + """ + Replace the field with the result of an expression. Produce the original field name with the expression result. + """ + + action: Literal['apply-expression'] = 'apply-expression' + expression: Expression | None = None + class LoadTableResult(BaseModel): """ @@ -2160,6 +2269,8 @@ class PlanTableScanResult( TableMetadata.model_rebuild() ViewMetadata.model_rebuild() AddSchemaUpdate.model_rebuild() +ReadRestrictions.model_rebuild() +ApplyExpression.model_rebuild() ScanTasks.model_rebuild() CommitTableRequest.model_rebuild() CommitViewRequest.model_rebuild() diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index bb1ec2064951..2b2cf2c3c356 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3678,25 +3678,24 @@ components: properties: required-column-projections: description: > - A list of columns that require specific projections or transforms to be applied. + A list of columns that require specific actions to be applied when reading. If this property is absent, a reader MAY access all columns of the table as-is without any mandatory transformations. If this property is present, each listed column MUST have its specified - projection or action applied. Columns not listed in required-column-projections + action applied. Columns not listed in required-column-projections are not subject to any read restrictions. When this list is present: 1. For each column listed in required-column-projections, the reader MUST apply - the specified action or transform. + the specified action before returning values for that column. - 2. The reader MUST apply the action and replace all references to the underlying - column with the transformed value. For example, if the action specifies - truncate[4](cc), the reader MUST project it as truncate[4](cc) AS cc, and all - references to cc during query evaluation (including in required-row-filter) - MUST resolve to this transformed alias. + 2. The reader MUST replace all output references to the column with the result + of the action, presenting the result under the original column name. For + example, if the action for column cc is mask-alphanum, the reader MUST + return the masked value as cc in the query output. 3. Columns not listed in required-column-projections MAY be projected normally by the reader without any mandatory transformations. @@ -3704,10 +3703,11 @@ components: 4. A column MUST appear at most once in required-column-projections. 5. If a projected column's action cannot be evaluated by the reader, - the reader MUST fail rather than ignore or skip the transform. + the reader MUST fail rather than ignore or skip the action. - 6. The data type of a projected column MUST match the data type defined for - the transform result. + 6. Each action defines the output type for its column. For all predefined + actions except apply-expression, the output type matches the input column + type. For apply-expression, the output type is determined by the expression. type: array items: @@ -3731,10 +3731,16 @@ components: discriminator: propertyName: action mapping: - mask-hash-sha256: '#/components/schemas/MaskHashSha256' + mask-alphanum: '#/components/schemas/MaskAlphanum' + mask-to-default: '#/components/schemas/MaskToDefault' replace-with-null: '#/components/schemas/ReplaceWithNull' - mask-alphanumeric: '#/components/schemas/MaskAlphanumeric' - apply-transform: '#/components/schemas/ApplyTransform' + show-first-4: '#/components/schemas/ShowFirst4' + show-last-4: '#/components/schemas/ShowLast4' + truncate-to-year: '#/components/schemas/TruncateToYear' + truncate-to-month: '#/components/schemas/TruncateToMonth' + sha-256-global: '#/components/schemas/Sha256Global' + sha-256-query-local: '#/components/schemas/Sha256QueryLocal' + apply-expression: '#/components/schemas/ApplyExpression' type: object required: - action @@ -3746,24 +3752,74 @@ components: type: integer description: field id of the column being projected. - MaskHashSha256: - description: | - Mask the data of the column by applying SHA-256. - The input must be UTF-8 encoded bytes of the column value. - The SHA-256 digest is represented as a lowercase hexadecimal string. - Engines must follow this procedure to ensure consistency: - 1. Convert the column value to a UTF-8 byte array. - 2. Apply the SHA-256 algorithm as specified in NIST FIPS 180-4. - 3. Convert the resulting 32-byte digest to a 64-character lowercase hexadecimal string. + MaskAlphanum: + description: > + Redacts the column value character by character using the following rules: + + - Digits (U+0030–U+0039, 0-9) are replaced with 'n' + - The following punctuation characters are kept as-is: + U+0028 '(' LEFT PARENTHESIS + U+0029 ')' RIGHT PARENTHESIS + U+002C ',' COMMA + U+002E '.' FULL STOP + U+002D '-' HYPHEN-MINUS + U+0040 '@' COMMERCIAL AT + - All other Unicode characters (including letters, whitespace, and any punctuation + not listed above) are replaced with 'x' + + For example: "prashant010696@gmail.com" → "xxxxxxxxnnnnnn@xxxxx.xxx" + + Applicable to: string + allOf: + - $ref: '#/components/schemas/Action' + properties: + action: + type: string + const: "mask-alphanum" + + MaskToDefault: + description: > + Replaces the column value with a predefined type-specific default that conceals + the original data while preserving type compatibility. Engines MUST use exactly + the values listed below to ensure consistency across implementations. + + Default values by type: + - boolean: false + - int: 999999999 + - long: 999999999 + - float: 0.0 + - double: 0.0 + - decimal(p, s): 0 (zero with s digits after the decimal point, e.g. 0.00 for decimal(p,2)) + - string: "XXXXXXXX" + - date: 9999-12-31 + - time: 00:00:00 + - timestamp: 9999-12-31T00:00:00 + - timestamptz: 9999-12-31T00:00:00+00:00 + - timestamp_ns: 9999-12-31T00:00:00.000000000 + - timestamptz_ns: 9999-12-31T00:00:00.000000000+00:00 + - uuid: 00000000-0000-0000-0000-000000000000 + - fixed(n): n zero bytes + - binary: empty byte sequence + - variant: {"masked": true} + - geometry: POINT EMPTY + - geography: POINT EMPTY + - list: empty list [] + - map: empty map {} + - struct: struct with each field set to its type-specific default (applied recursively) + + Applicable to: all data types allOf: - $ref: '#/components/schemas/Action' properties: action: type: string - const: "mask-hash-sha256" + const: "mask-to-default" ReplaceWithNull: - description: Masks data by replacing it with a NULL value. + description: > + Replaces the entire column value with NULL. + + Applicable to: all nullable types allOf: - $ref: '#/components/schemas/Action' properties: @@ -3771,25 +3827,124 @@ components: type: string const: "replace-with-null" - MaskAlphanumeric: - description: mask all alphabetic characters with 'x' and numeric characters with 'n' + ShowFirst4: + description: > + Preserves the first 4 characters of the column value and redacts the remainder + using mask-alphanum rules (see MaskAlphanum for the exact character rules). + Values with 4 or fewer characters are returned unchanged. + + For example: "prashant010696@gmail.com" → "prasxxxxnnnnnn@xxxxx.xxx" + + Applicable to: string allOf: - $ref: '#/components/schemas/Action' properties: action: type: string - const: "mask-alphanumeric" + const: "show-first-4" + + ShowLast4: + description: > + Redacts all characters except the last 4 using mask-alphanum rules + (see MaskAlphanum for the exact character rules). + Values with 4 or fewer characters are returned unchanged. - ApplyTransform: - description: Replace the field with the result of a transform expression. Produce the original field name with the transformed values. + For example: "4111-1111-1111-4444" → "nnnn-nnnn-nnnn-4444" + + Applicable to: string allOf: - $ref: '#/components/schemas/Action' properties: action: type: string - const: "apply-transform" - term: - $ref: '#/components/schemas/Term' + const: "show-last-4" + + TruncateToYear: + description: > + Truncates the column value to year precision, setting month, day, and time components + to their minimum values. The output type matches the input type. + + For example: 2024-07-15 → 2024-01-01 + + Applicable to: date, timestamp, timestamptz, timestamp_ns, timestamptz_ns + allOf: + - $ref: '#/components/schemas/Action' + properties: + action: + type: string + const: "truncate-to-year" + + TruncateToMonth: + description: > + Truncates the column value to year and month precision, setting day and time components + to their minimum values. The output type matches the input type. + + For example: 2024-07-15 → 2024-07-01 + + Applicable to: date, timestamp, timestamptz, timestamp_ns, timestamptz_ns + allOf: + - $ref: '#/components/schemas/Action' + properties: + action: + type: string + const: "truncate-to-month" + + Sha256Global: + description: | + Applies SHA-256 as specified in NIST FIPS 180-4. Deterministic across all queries + and engines — the same input always produces the same output. + + Input-to-bytes encoding by type: + - string: UTF-8 encoded bytes + - int: 4 bytes, little-endian + - long: 8 bytes, little-endian + - uuid: 16 bytes, big-endian (RFC 4122 byte order) + - binary: raw bytes as-is + + Output encoding by type: + - string: 64-character lowercase hexadecimal string + - int: first 4 bytes of the digest, read as a little-endian int + - long: first 8 bytes of the digest, read as a little-endian long + - uuid: all 16 bytes of the digest, interpreted as a UUID (RFC 4122 byte order) + - binary: the full 32-byte raw SHA-256 digest, output type is binary(32) + + Applicable to: string, int, long, uuid, binary + allOf: + - $ref: '#/components/schemas/Action' + properties: + action: + type: string + const: "sha-256-global" + + Sha256QueryLocal: + description: | + Applies SHA-256 with a per-query random salt, making the output non-deterministic + across queries while remaining consistent within a single query. + + The engine MUST generate a cryptographically random salt for each query and apply it as: + SHA-256(salt_bytes || canonical_bytes) + where canonical_bytes follows the same encoding rules as sha-256-global. + + Output encoding follows the same rules as sha-256-global. + + Applicable to: string, int, long, uuid, binary + allOf: + - $ref: '#/components/schemas/Action' + properties: + action: + type: string + const: "sha-256-query-local" + + ApplyExpression: + description: Replace the field with the result of an expression. Produce the original field name with the expression result. + allOf: + - $ref: '#/components/schemas/Action' + properties: + action: + type: string + const: "apply-expression" + expression: + $ref: '#/components/schemas/Expression' LoadCredentialsResponse: type: object From 53f3841628bd9e4d3743c831a5e986b8f0fffb5b Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Sat, 18 Apr 2026 00:24:20 -0700 Subject: [PATCH 20/45] Tighten spec precision for catalog and engine implementers - Mark ApplyExpression.expression as required - Fix SHA-256 binary output type from binary(32) to binary - Add NULL-in/NULL-out global rule for all actions - Specify Unicode code point semantics for masking actions - Add signed two's complement for SHA-256 int/long output - Add UTC truncation rule for timestamptz types - Clarify empty ReadRestrictions equivalence and server type validation - Specify minimum 16-byte salt for sha-256-query-local - Clarify unrecognized action types trigger MUST-fail - Add "Applicable to: all data types" for ApplyExpression - Regenerate Python from updated YAML spec --- open-api/rest-catalog-open-api.py | 27 ++++++++++--------- open-api/rest-catalog-open-api.yaml | 40 ++++++++++++++++++++--------- 2 files changed, 43 insertions(+), 24 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index 2dc8f6b8be91..c838a86fb537 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -597,7 +597,7 @@ class Action(BaseModel): class MaskAlphanum(Action): """ - Redacts the column value character by character using the following rules: + Redacts the column value Unicode code point by code point using the following rules: - Digits (U+0030–U+0039, 0-9) are replaced with 'n' - The following punctuation characters are kept as-is: U+0028 '(' LEFT PARENTHESIS U+0029 ')' RIGHT PARENTHESIS @@ -639,7 +639,7 @@ class ReplaceWithNull(Action): class ShowFirst4(Action): """ - Preserves the first 4 characters of the column value and redacts the remainder using mask-alphanum rules (see MaskAlphanum for the exact character rules). Values with 4 or fewer characters are returned unchanged. + Preserves the first 4 Unicode code points of the column value and redacts the remainder using mask-alphanum rules (see MaskAlphanum for the exact character rules). Values with 4 or fewer Unicode code points are returned unchanged. For example: "prashant010696@gmail.com" → "prasxxxxnnnnnn@xxxxx.xxx" Applicable to: string @@ -650,7 +650,7 @@ class ShowFirst4(Action): class ShowLast4(Action): """ - Redacts all characters except the last 4 using mask-alphanum rules (see MaskAlphanum for the exact character rules). Values with 4 or fewer characters are returned unchanged. + Redacts all Unicode code points except the last 4 using mask-alphanum rules (see MaskAlphanum for the exact character rules). Values with 4 or fewer Unicode code points are returned unchanged. For example: "4111-1111-1111-4444" → "nnnn-nnnn-nnnn-4444" Applicable to: string @@ -662,7 +662,7 @@ class ShowLast4(Action): class TruncateToYear(Action): """ Truncates the column value to year precision, setting month, day, and time components to their minimum values. The output type matches the input type. - For example: 2024-07-15 → 2024-01-01 + For example: 2024-07-15 → 2024-01-01 For timestamptz and timestamptz_ns, truncation is performed in UTC. Applicable to: date, timestamp, timestamptz, timestamp_ns, timestamptz_ns """ @@ -673,7 +673,7 @@ class TruncateToYear(Action): class TruncateToMonth(Action): """ Truncates the column value to year and month precision, setting day and time components to their minimum values. The output type matches the input type. - For example: 2024-07-15 → 2024-07-01 + For example: 2024-07-15 → 2024-07-01 For timestamptz and timestamptz_ns, truncation is performed in UTC. Applicable to: date, timestamp, timestamptz, timestamp_ns, timestamptz_ns """ @@ -695,10 +695,10 @@ class Sha256Global(Action): Output encoding by type: - string: 64-character lowercase hexadecimal string - - int: first 4 bytes of the digest, read as a little-endian int - - long: first 8 bytes of the digest, read as a little-endian long + - int: first 4 bytes of the digest, read as a signed two's complement little-endian int + - long: first 8 bytes of the digest, read as a signed two's complement little-endian long - uuid: all 16 bytes of the digest, interpreted as a UUID (RFC 4122 byte order) - - binary: the full 32-byte raw SHA-256 digest, output type is binary(32) + - binary: the full 32-byte raw SHA-256 digest Applicable to: string, int, long, uuid, binary @@ -712,7 +712,7 @@ class Sha256QueryLocal(Action): Applies SHA-256 with a per-query random salt, making the output non-deterministic across queries while remaining consistent within a single query. - The engine MUST generate a cryptographically random salt for each query and apply it as: + The engine MUST generate a cryptographically random salt of at least 16 bytes for each query and apply it as: SHA-256(salt_bytes || canonical_bytes) where canonical_bytes follows the same encoding rules as sha-256-global. @@ -1667,6 +1667,7 @@ class ReadRestrictions(BaseModel): Read restrictions for a table, including column projections and row filter expressions. A client MUST enforce the restrictions defined in this object when reading data from the table. These restrictions apply only to the authenticated principal, user, or account associated with the request. They MUST NOT be interpreted as global policy and MUST NOT be applied beyond the entity identified by the Authentication header (or other applicable authentication mechanism). + If both properties are absent or empty, the ReadRestrictions object imposes no restrictions and is equivalent to the field being absent from the response. A server MUST NOT return an action for a column whose type is not listed in that action's "Applicable to" set. For all actions, if the input column value is NULL, the output MUST be NULL. """ @@ -1687,7 +1688,7 @@ class ReadRestrictions(BaseModel): ) = Field( None, alias='required-column-projections', - description="A list of columns that require specific actions to be applied when reading.\nIf this property is absent, a reader MAY access all columns of the table as-is without any mandatory transformations.\nIf this property is present, each listed column MUST have its specified action applied. Columns not listed in required-column-projections are not subject to any read restrictions.\nWhen this list is present:\n1. For each column listed in required-column-projections, the reader MUST apply\n the specified action before returning values for that column.\n\n2. The reader MUST replace all output references to the column with the result\n of the action, presenting the result under the original column name. For\n example, if the action for column cc is mask-alphanum, the reader MUST\n return the masked value as cc in the query output.\n\n3. Columns not listed in required-column-projections MAY be projected normally\n by the reader without any mandatory transformations.\n\n4. A column MUST appear at most once in required-column-projections.\n5. If a projected column's action cannot be evaluated by the reader,\n the reader MUST fail rather than ignore or skip the action.\n\n6. Each action defines the output type for its column. For all predefined\n actions except apply-expression, the output type matches the input column\n type. For apply-expression, the output type is determined by the expression.\n", + description="A list of columns that require specific actions to be applied when reading.\nIf this property is absent, a reader MAY access all columns of the table as-is without any mandatory transformations.\nIf this property is present, each listed column MUST have its specified action applied. Columns not listed in required-column-projections are not subject to any read restrictions.\nWhen this list is present:\n1. For each column listed in required-column-projections, the reader MUST apply\n the specified action before returning values for that column.\n\n2. The reader MUST replace all output references to the column with the result\n of the action, presenting the result under the original column name. For\n example, if the action for column cc is mask-alphanum, the reader MUST\n return the masked value as cc in the query output.\n\n3. Columns not listed in required-column-projections MAY be projected normally\n by the reader without any mandatory transformations.\n\n4. A column MUST appear at most once in required-column-projections.\n5. If a projected column's action cannot be evaluated by the reader\n (including unrecognized action types), the reader MUST fail rather than\n ignore or skip the action.\n\n6. Each action defines the output type for its column. For all predefined\n actions except apply-expression, the output type matches the input column\n type. For apply-expression, the output type is determined by the expression.\n", ) required_row_filter: Expression | None = Field( None, @@ -1699,10 +1700,12 @@ class ReadRestrictions(BaseModel): class ApplyExpression(Action): """ Replace the field with the result of an expression. Produce the original field name with the expression result. + Applicable to: all data types + """ - action: Literal['apply-expression'] = 'apply-expression' - expression: Expression | None = None + action: Literal['apply-expression'] + expression: Expression class LoadTableResult(BaseModel): diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index 2b2cf2c3c356..26357bd429d8 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3675,6 +3675,12 @@ components: associated with the request. They MUST NOT be interpreted as global policy and MUST NOT be applied beyond the entity identified by the Authentication header (or other applicable authentication mechanism). + + If both properties are absent or empty, the ReadRestrictions object imposes no + restrictions and is equivalent to the field being absent from the response. + A server MUST NOT return an action for a column whose type is not listed in + that action's "Applicable to" set. + For all actions, if the input column value is NULL, the output MUST be NULL. properties: required-column-projections: description: > @@ -3702,8 +3708,9 @@ components: 4. A column MUST appear at most once in required-column-projections. - 5. If a projected column's action cannot be evaluated by the reader, - the reader MUST fail rather than ignore or skip the action. + 5. If a projected column's action cannot be evaluated by the reader + (including unrecognized action types), the reader MUST fail rather than + ignore or skip the action. 6. Each action defines the output type for its column. For all predefined actions except apply-expression, the output type matches the input column @@ -3754,7 +3761,7 @@ components: MaskAlphanum: description: > - Redacts the column value character by character using the following rules: + Redacts the column value Unicode code point by code point using the following rules: - Digits (U+0030–U+0039, 0-9) are replaced with 'n' - The following punctuation characters are kept as-is: @@ -3829,9 +3836,9 @@ components: ShowFirst4: description: > - Preserves the first 4 characters of the column value and redacts the remainder + Preserves the first 4 Unicode code points of the column value and redacts the remainder using mask-alphanum rules (see MaskAlphanum for the exact character rules). - Values with 4 or fewer characters are returned unchanged. + Values with 4 or fewer Unicode code points are returned unchanged. For example: "prashant010696@gmail.com" → "prasxxxxnnnnnn@xxxxx.xxx" @@ -3845,9 +3852,9 @@ components: ShowLast4: description: > - Redacts all characters except the last 4 using mask-alphanum rules + Redacts all Unicode code points except the last 4 using mask-alphanum rules (see MaskAlphanum for the exact character rules). - Values with 4 or fewer characters are returned unchanged. + Values with 4 or fewer Unicode code points are returned unchanged. For example: "4111-1111-1111-4444" → "nnnn-nnnn-nnnn-4444" @@ -3865,6 +3872,7 @@ components: to their minimum values. The output type matches the input type. For example: 2024-07-15 → 2024-01-01 + For timestamptz and timestamptz_ns, truncation is performed in UTC. Applicable to: date, timestamp, timestamptz, timestamp_ns, timestamptz_ns allOf: @@ -3880,6 +3888,7 @@ components: to their minimum values. The output type matches the input type. For example: 2024-07-15 → 2024-07-01 + For timestamptz and timestamptz_ns, truncation is performed in UTC. Applicable to: date, timestamp, timestamptz, timestamp_ns, timestamptz_ns allOf: @@ -3903,10 +3912,10 @@ components: Output encoding by type: - string: 64-character lowercase hexadecimal string - - int: first 4 bytes of the digest, read as a little-endian int - - long: first 8 bytes of the digest, read as a little-endian long + - int: first 4 bytes of the digest, read as a signed two's complement little-endian int + - long: first 8 bytes of the digest, read as a signed two's complement little-endian long - uuid: all 16 bytes of the digest, interpreted as a UUID (RFC 4122 byte order) - - binary: the full 32-byte raw SHA-256 digest, output type is binary(32) + - binary: the full 32-byte raw SHA-256 digest Applicable to: string, int, long, uuid, binary allOf: @@ -3921,7 +3930,7 @@ components: Applies SHA-256 with a per-query random salt, making the output non-deterministic across queries while remaining consistent within a single query. - The engine MUST generate a cryptographically random salt for each query and apply it as: + The engine MUST generate a cryptographically random salt of at least 16 bytes for each query and apply it as: SHA-256(salt_bytes || canonical_bytes) where canonical_bytes follows the same encoding rules as sha-256-global. @@ -3936,9 +3945,16 @@ components: const: "sha-256-query-local" ApplyExpression: - description: Replace the field with the result of an expression. Produce the original field name with the expression result. + description: > + Replace the field with the result of an expression. Produce the original field name + with the expression result. + + Applicable to: all data types allOf: - $ref: '#/components/schemas/Action' + required: + - action + - expression properties: action: type: string From a8d350340d32ef16646e14fbcc297c37e64d5351 Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Sat, 18 Apr 2026 01:12:34 -0700 Subject: [PATCH 21/45] Remove uuid from SHA-256 action applicable types SHA-256 digest truncated to 16 bytes does not produce valid RFC 4122 UUIDs (version and variant bits are not set), so uuid is removed from the input encoding, output encoding, and applicable types for both sha-256-global and sha-256-query-local actions. --- open-api/rest-catalog-open-api.py | 6 ++---- open-api/rest-catalog-open-api.yaml | 6 ++---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index c838a86fb537..41e427c87f84 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -690,17 +690,15 @@ class Sha256Global(Action): - string: UTF-8 encoded bytes - int: 4 bytes, little-endian - long: 8 bytes, little-endian - - uuid: 16 bytes, big-endian (RFC 4122 byte order) - binary: raw bytes as-is Output encoding by type: - string: 64-character lowercase hexadecimal string - int: first 4 bytes of the digest, read as a signed two's complement little-endian int - long: first 8 bytes of the digest, read as a signed two's complement little-endian long - - uuid: all 16 bytes of the digest, interpreted as a UUID (RFC 4122 byte order) - binary: the full 32-byte raw SHA-256 digest - Applicable to: string, int, long, uuid, binary + Applicable to: string, int, long, binary """ @@ -718,7 +716,7 @@ class Sha256QueryLocal(Action): Output encoding follows the same rules as sha-256-global. - Applicable to: string, int, long, uuid, binary + Applicable to: string, int, long, binary """ diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index 26357bd429d8..5bd9a5978f3b 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3907,17 +3907,15 @@ components: - string: UTF-8 encoded bytes - int: 4 bytes, little-endian - long: 8 bytes, little-endian - - uuid: 16 bytes, big-endian (RFC 4122 byte order) - binary: raw bytes as-is Output encoding by type: - string: 64-character lowercase hexadecimal string - int: first 4 bytes of the digest, read as a signed two's complement little-endian int - long: first 8 bytes of the digest, read as a signed two's complement little-endian long - - uuid: all 16 bytes of the digest, interpreted as a UUID (RFC 4122 byte order) - binary: the full 32-byte raw SHA-256 digest - Applicable to: string, int, long, uuid, binary + Applicable to: string, int, long, binary allOf: - $ref: '#/components/schemas/Action' properties: @@ -3936,7 +3934,7 @@ components: Output encoding follows the same rules as sha-256-global. - Applicable to: string, int, long, uuid, binary + Applicable to: string, int, long, binary allOf: - $ref: '#/components/schemas/Action' properties: From 570573dc5c2a3ba759802c5f10f219524dbdeebe Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Tue, 21 Apr 2026 17:58:54 -0700 Subject: [PATCH 22/45] Fix timestamp_ns mask-to-default overflow in spec The spec mandated 9999-12-31T00:00:00 as the mask-to-default sentinel for timestamp_ns and timestamptz_ns, but that value overflows a 64-bit signed integer as nanoseconds from epoch (max representable is ~2262-04-11). Engines MUST use exactly the listed default, so specifying an unrepresentable value broke cross-engine consistency. Use 2261-12-31T00:00:00 as the closest clean far-future sentinel that fits. Add a note explaining the overflow so implementers don't mistake it for a typo. --- open-api/rest-catalog-open-api.py | 3 ++- open-api/rest-catalog-open-api.yaml | 8 ++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index 41e427c87f84..fd9ef8eb3b11 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -619,7 +619,8 @@ class MaskAlphanum(Action): class MaskToDefault(Action): """ Replaces the column value with a predefined type-specific default that conceals the original data while preserving type compatibility. Engines MUST use exactly the values listed below to ensure consistency across implementations. - Default values by type: - boolean: false - int: 999999999 - long: 999999999 - float: 0.0 - double: 0.0 - decimal(p, s): 0 (zero with s digits after the decimal point, e.g. 0.00 for decimal(p,2)) - string: "XXXXXXXX" - date: 9999-12-31 - time: 00:00:00 - timestamp: 9999-12-31T00:00:00 - timestamptz: 9999-12-31T00:00:00+00:00 - timestamp_ns: 9999-12-31T00:00:00.000000000 - timestamptz_ns: 9999-12-31T00:00:00.000000000+00:00 - uuid: 00000000-0000-0000-0000-000000000000 - fixed(n): n zero bytes - binary: empty byte sequence - variant: {"masked": true} - geometry: POINT EMPTY - geography: POINT EMPTY - list: empty list [] - map: empty map {} - struct: struct with each field set to its type-specific default (applied recursively) + Default values by type: - boolean: false - int: 999999999 - long: 999999999 - float: 0.0 - double: 0.0 - decimal(p, s): 0 (zero with s digits after the decimal point, e.g. 0.00 for decimal(p,2)) - string: "XXXXXXXX" - date: 9999-12-31 - time: 00:00:00 - timestamp: 9999-12-31T00:00:00 - timestamptz: 9999-12-31T00:00:00+00:00 - timestamp_ns: 2261-12-31T00:00:00.000000000 - timestamptz_ns: 2261-12-31T00:00:00.000000000+00:00 - uuid: 00000000-0000-0000-0000-000000000000 - fixed(n): n zero bytes - binary: empty byte sequence - variant: {"masked": true} - geometry: POINT EMPTY - geography: POINT EMPTY - list: empty list [] - map: empty map {} - struct: struct with each field set to its type-specific default (applied recursively) + Note: nanosecond-precision timestamps (timestamp_ns, timestamptz_ns) cannot represent 9999-12-31 because nanoseconds from epoch overflows a 64-bit signed integer past approximately 2262-04-11; 2261-12-31 is used as the closest clean far-future sentinel. Applicable to: all data types """ diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index 5bd9a5978f3b..b308b32a0d22 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3802,8 +3802,8 @@ components: - time: 00:00:00 - timestamp: 9999-12-31T00:00:00 - timestamptz: 9999-12-31T00:00:00+00:00 - - timestamp_ns: 9999-12-31T00:00:00.000000000 - - timestamptz_ns: 9999-12-31T00:00:00.000000000+00:00 + - timestamp_ns: 2261-12-31T00:00:00.000000000 + - timestamptz_ns: 2261-12-31T00:00:00.000000000+00:00 - uuid: 00000000-0000-0000-0000-000000000000 - fixed(n): n zero bytes - binary: empty byte sequence @@ -3814,6 +3814,10 @@ components: - map: empty map {} - struct: struct with each field set to its type-specific default (applied recursively) + Note: nanosecond-precision timestamps (timestamp_ns, timestamptz_ns) cannot represent + 9999-12-31 because nanoseconds from epoch overflows a 64-bit signed integer past + approximately 2262-04-11; 2261-12-31 is used as the closest clean far-future sentinel. + Applicable to: all data types allOf: - $ref: '#/components/schemas/Action' From a0e0d27fba726ef147e5dac0bc539f8bb3abc035 Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Sat, 2 May 2026 16:25:17 -0700 Subject: [PATCH 23/45] Open API: Address FGAC spec review feedback - Row filter: require boolean evaluation (resolves @nicholasdoyleg) - Add struct coexistence rule: no subfield projections when struct is projected - Rename mask-to-default to mask-to-fixed-value - Adopt zero-of-type defaults (epoch dates/timestamps, int/long=0, variant={}) while keeping string="XXXXXXXX" for observability - Drop nanos-overflow note (epoch fits int64 nanos) --- open-api/rest-catalog-open-api.py | 14 ++++----- open-api/rest-catalog-open-api.yaml | 44 +++++++++++++++-------------- 2 files changed, 30 insertions(+), 28 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index fd9ef8eb3b11..17a0b27f25e9 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -616,16 +616,15 @@ class MaskAlphanum(Action): action: Literal['mask-alphanum'] = 'mask-alphanum' -class MaskToDefault(Action): +class MaskToFixedValue(Action): """ - Replaces the column value with a predefined type-specific default that conceals the original data while preserving type compatibility. Engines MUST use exactly the values listed below to ensure consistency across implementations. - Default values by type: - boolean: false - int: 999999999 - long: 999999999 - float: 0.0 - double: 0.0 - decimal(p, s): 0 (zero with s digits after the decimal point, e.g. 0.00 for decimal(p,2)) - string: "XXXXXXXX" - date: 9999-12-31 - time: 00:00:00 - timestamp: 9999-12-31T00:00:00 - timestamptz: 9999-12-31T00:00:00+00:00 - timestamp_ns: 2261-12-31T00:00:00.000000000 - timestamptz_ns: 2261-12-31T00:00:00.000000000+00:00 - uuid: 00000000-0000-0000-0000-000000000000 - fixed(n): n zero bytes - binary: empty byte sequence - variant: {"masked": true} - geometry: POINT EMPTY - geography: POINT EMPTY - list: empty list [] - map: empty map {} - struct: struct with each field set to its type-specific default (applied recursively) - Note: nanosecond-precision timestamps (timestamp_ns, timestamptz_ns) cannot represent 9999-12-31 because nanoseconds from epoch overflows a 64-bit signed integer past approximately 2262-04-11; 2261-12-31 is used as the closest clean far-future sentinel. + Replaces the column value with a predefined type-specific fixed value. Engines MUST use exactly the values listed below to ensure consistency across implementations. + Fixed values by type: - boolean: false - int: 0 - long: 0 - float: 0.0 - double: 0.0 - decimal(p, s): 0 (zero with s digits after the decimal point, e.g. 0.00 for decimal(p,2)) - string: "XXXXXXXX" - date: 1970-01-01 - time: 00:00:00 - timestamp: 1970-01-01T00:00:00 - timestamptz: 1970-01-01T00:00:00+00:00 - timestamp_ns: 1970-01-01T00:00:00.000000000 - timestamptz_ns: 1970-01-01T00:00:00.000000000+00:00 - uuid: 00000000-0000-0000-0000-000000000000 - fixed(n): n zero bytes - binary: empty byte sequence - variant: {} - geometry: POINT EMPTY - geography: POINT EMPTY - list: empty list [] - map: empty map {} - struct: struct with each field set to its type-specific default (applied recursively) Applicable to: all data types """ - action: Literal['mask-to-default'] = 'mask-to-default' + action: Literal['mask-to-fixed-value'] = 'mask-to-fixed-value' class ReplaceWithNull(Action): @@ -1667,13 +1666,14 @@ class ReadRestrictions(BaseModel): A client MUST enforce the restrictions defined in this object when reading data from the table. These restrictions apply only to the authenticated principal, user, or account associated with the request. They MUST NOT be interpreted as global policy and MUST NOT be applied beyond the entity identified by the Authentication header (or other applicable authentication mechanism). If both properties are absent or empty, the ReadRestrictions object imposes no restrictions and is equivalent to the field being absent from the response. A server MUST NOT return an action for a column whose type is not listed in that action's "Applicable to" set. For all actions, if the input column value is NULL, the output MUST be NULL. + If a column projection targets a struct-typed field, other column projections in the same ReadRestrictions MUST NOT target any of that struct's subfields (at any depth). This avoids ambiguity about which action governs a given leaf value. """ required_column_projections: ( list[ MaskAlphanum - | MaskToDefault + | MaskToFixedValue | ReplaceWithNull | ShowFirst4 | ShowLast4 @@ -1692,7 +1692,7 @@ class ReadRestrictions(BaseModel): required_row_filter: Expression | None = Field( None, alias='required-row-filter', - description='An expression that filters rows in the table that the authenticated principal does not have access to.\n1. A reader MUST discard any row for which the filter evaluates to false or null, and\n no information derived from discarded rows MAY be included in the query result.\n\n2. Row filters MUST be evaluated against the original, untransformed column values.\n Required projections MUST be applied only after row filters are applied.\n\n3. If a client cannot interpret or evaluate a provided filter expression, it MUST fail.\n4. If this property is absent, null, or always true then no mandatory filtering is required.\n', + description='An expression that filters rows in the table that the authenticated principal does not have access to.\n1. The expression MUST evaluate to a boolean. A reader MUST discard any row for which\n the filter evaluates to FALSE, and no information derived from discarded rows\n MAY be included in the query result.\n\n2. Row filters MUST be evaluated against the original, untransformed column values.\n Required projections MUST be applied only after row filters are applied.\n\n3. If a client cannot interpret or evaluate a provided filter expression, it MUST fail.\n4. If this property is absent, null, or always true then no mandatory filtering is required.\n', ) diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index b308b32a0d22..1714116c4a90 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3681,6 +3681,11 @@ components: A server MUST NOT return an action for a column whose type is not listed in that action's "Applicable to" set. For all actions, if the input column value is NULL, the output MUST be NULL. + + If a column projection targets a struct-typed field, other column projections + in the same ReadRestrictions MUST NOT target any of that struct's subfields + (at any depth). This avoids ambiguity about which action governs a given + leaf value. properties: required-column-projections: description: > @@ -3723,8 +3728,9 @@ components: description: > An expression that filters rows in the table that the authenticated principal does not have access to. - 1. A reader MUST discard any row for which the filter evaluates to false or null, and - no information derived from discarded rows MAY be included in the query result. + 1. The expression MUST evaluate to a boolean. A reader MUST discard any row for which + the filter evaluates to FALSE, and no information derived from discarded rows + MAY be included in the query result. 2. Row filters MUST be evaluated against the original, untransformed column values. Required projections MUST be applied only after row filters are applied. @@ -3739,7 +3745,7 @@ components: propertyName: action mapping: mask-alphanum: '#/components/schemas/MaskAlphanum' - mask-to-default: '#/components/schemas/MaskToDefault' + mask-to-fixed-value: '#/components/schemas/MaskToFixedValue' replace-with-null: '#/components/schemas/ReplaceWithNull' show-first-4: '#/components/schemas/ShowFirst4' show-last-4: '#/components/schemas/ShowLast4' @@ -3784,47 +3790,43 @@ components: type: string const: "mask-alphanum" - MaskToDefault: + MaskToFixedValue: description: > - Replaces the column value with a predefined type-specific default that conceals - the original data while preserving type compatibility. Engines MUST use exactly - the values listed below to ensure consistency across implementations. + Replaces the column value with a predefined type-specific fixed value. + Engines MUST use exactly the values listed below to ensure consistency + across implementations. - Default values by type: + Fixed values by type: - boolean: false - - int: 999999999 - - long: 999999999 + - int: 0 + - long: 0 - float: 0.0 - double: 0.0 - decimal(p, s): 0 (zero with s digits after the decimal point, e.g. 0.00 for decimal(p,2)) - string: "XXXXXXXX" - - date: 9999-12-31 + - date: 1970-01-01 - time: 00:00:00 - - timestamp: 9999-12-31T00:00:00 - - timestamptz: 9999-12-31T00:00:00+00:00 - - timestamp_ns: 2261-12-31T00:00:00.000000000 - - timestamptz_ns: 2261-12-31T00:00:00.000000000+00:00 + - timestamp: 1970-01-01T00:00:00 + - timestamptz: 1970-01-01T00:00:00+00:00 + - timestamp_ns: 1970-01-01T00:00:00.000000000 + - timestamptz_ns: 1970-01-01T00:00:00.000000000+00:00 - uuid: 00000000-0000-0000-0000-000000000000 - fixed(n): n zero bytes - binary: empty byte sequence - - variant: {"masked": true} + - variant: {} - geometry: POINT EMPTY - geography: POINT EMPTY - list: empty list [] - map: empty map {} - struct: struct with each field set to its type-specific default (applied recursively) - Note: nanosecond-precision timestamps (timestamp_ns, timestamptz_ns) cannot represent - 9999-12-31 because nanoseconds from epoch overflows a 64-bit signed integer past - approximately 2262-04-11; 2261-12-31 is used as the closest clean far-future sentinel. - Applicable to: all data types allOf: - $ref: '#/components/schemas/Action' properties: action: type: string - const: "mask-to-default" + const: "mask-to-fixed-value" ReplaceWithNull: description: > From 47258ec4b6d32a1f0a39a6f0ba2313af66c8e4fe Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Sun, 17 May 2026 11:45:24 -0700 Subject: [PATCH 24/45] Open API: Refine FGAC read restrictions spec Address review feedback and sync conclusions on the ReadRestrictions schema and its actions: - Remove apply-expression action (deferred until expression-language extension lands). - Drop the global "NULL -> NULL" rule and write per-action NULL behavior into all 9 actions. mask-to-fixed-value emits the type-specific default for NULL inputs (NULL not preserved); the other 8 actions preserve NULL. - ReplaceWithNull: server MUST NOT return for non-nullable columns. - Tighten fail-mode wording to fail-closed semantics. Reader MUST fail the query with an error and MUST NOT silently return raw, partial, or empty results to mask the failure (projection rule 5 and row-filter rule 3). - Unify the normative actor as "reader" for per-row behavior, keep "engine" for query-scoped behavior (salt generation), and define both terms inline. - Add a downstream-applicability sentence: restrictions apply to every read operation that uses metadata from the load response, including planTableScan calls. - Add a worked JSON example for ReadRestrictions. - Clarify row-filter rule 1: Iceberg expressions are 2-valued (TRUE or FALSE; never NULL). - Sha256QueryLocal: replace cryptographic concat notation (SHA-256(salt_bytes || canonical_bytes)) with plain prose so the spec is dialect-agnostic. - Wrap required-row-filter $ref in allOf so the description is no longer silently dropped by OpenAPI tooling. - Style/wording cleanup: capitalize "Field ID", clarify "two properties" wording. --- open-api/rest-catalog-open-api.py | 63 ++++++++++------ open-api/rest-catalog-open-api.yaml | 108 ++++++++++++++++++---------- 2 files changed, 110 insertions(+), 61 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index 17a0b27f25e9..43d327c5a6ef 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -591,7 +591,7 @@ class StorageCredential(BaseModel): class Action(BaseModel): action: str field_id: int = Field( - ..., alias='field-id', description='field id of the column being projected.' + ..., alias='field-id', description='Field ID of the column being projected.' ) @@ -609,6 +609,7 @@ class MaskAlphanum(Action): not listed above) are replaced with 'x' For example: "prashant010696@gmail.com" → "xxxxxxxxnnnnnn@xxxxx.xxx" + NULL input is preserved (NULL → NULL). Applicable to: string """ @@ -618,8 +619,9 @@ class MaskAlphanum(Action): class MaskToFixedValue(Action): """ - Replaces the column value with a predefined type-specific fixed value. Engines MUST use exactly the values listed below to ensure consistency across implementations. + Replaces the column value with a predefined type-specific fixed value. Readers MUST use exactly the values listed below to ensure consistency across implementations. Fixed values by type: - boolean: false - int: 0 - long: 0 - float: 0.0 - double: 0.0 - decimal(p, s): 0 (zero with s digits after the decimal point, e.g. 0.00 for decimal(p,2)) - string: "XXXXXXXX" - date: 1970-01-01 - time: 00:00:00 - timestamp: 1970-01-01T00:00:00 - timestamptz: 1970-01-01T00:00:00+00:00 - timestamp_ns: 1970-01-01T00:00:00.000000000 - timestamptz_ns: 1970-01-01T00:00:00.000000000+00:00 - uuid: 00000000-0000-0000-0000-000000000000 - fixed(n): n zero bytes - binary: empty byte sequence - variant: {} - geometry: POINT EMPTY - geography: POINT EMPTY - list: empty list [] - map: empty map {} - struct: struct with each field set to its type-specific default (applied recursively) + NULL input is also replaced with the type-specific fixed value; NULL is not preserved. Applicable to: all data types """ @@ -629,7 +631,7 @@ class MaskToFixedValue(Action): class ReplaceWithNull(Action): """ - Replaces the entire column value with NULL. + Replaces the entire column value with NULL. NULL input is preserved (NULL → NULL). A server MUST NOT return this action for a non-nullable (required) column. Applicable to: all nullable types """ @@ -641,6 +643,7 @@ class ShowFirst4(Action): """ Preserves the first 4 Unicode code points of the column value and redacts the remainder using mask-alphanum rules (see MaskAlphanum for the exact character rules). Values with 4 or fewer Unicode code points are returned unchanged. For example: "prashant010696@gmail.com" → "prasxxxxnnnnnn@xxxxx.xxx" + NULL input is preserved (NULL → NULL). Applicable to: string """ @@ -652,6 +655,7 @@ class ShowLast4(Action): """ Redacts all Unicode code points except the last 4 using mask-alphanum rules (see MaskAlphanum for the exact character rules). Values with 4 or fewer Unicode code points are returned unchanged. For example: "4111-1111-1111-4444" → "nnnn-nnnn-nnnn-4444" + NULL input is preserved (NULL → NULL). Applicable to: string """ @@ -663,6 +667,7 @@ class TruncateToYear(Action): """ Truncates the column value to year precision, setting month, day, and time components to their minimum values. The output type matches the input type. For example: 2024-07-15 → 2024-01-01 For timestamptz and timestamptz_ns, truncation is performed in UTC. + NULL input is preserved (NULL → NULL). Applicable to: date, timestamp, timestamptz, timestamp_ns, timestamptz_ns """ @@ -674,6 +679,7 @@ class TruncateToMonth(Action): """ Truncates the column value to year and month precision, setting day and time components to their minimum values. The output type matches the input type. For example: 2024-07-15 → 2024-07-01 For timestamptz and timestamptz_ns, truncation is performed in UTC. + NULL input is preserved (NULL → NULL). Applicable to: date, timestamp, timestamptz, timestamp_ns, timestamptz_ns """ @@ -698,6 +704,8 @@ class Sha256Global(Action): - long: first 8 bytes of the digest, read as a signed two's complement little-endian long - binary: the full 32-byte raw SHA-256 digest + NULL input is preserved (NULL → NULL). + Applicable to: string, int, long, binary """ @@ -710,12 +718,16 @@ class Sha256QueryLocal(Action): Applies SHA-256 with a per-query random salt, making the output non-deterministic across queries while remaining consistent within a single query. - The engine MUST generate a cryptographically random salt of at least 16 bytes for each query and apply it as: - SHA-256(salt_bytes || canonical_bytes) - where canonical_bytes follows the same encoding rules as sha-256-global. + The engine MUST generate a cryptographically random salt of at least 16 bytes for each query. + + For each column value, the engine MUST encode the value to bytes using + sha-256-global's input rules, prepend the per-query salt, and compute + SHA-256 over the result. Output encoding follows the same rules as sha-256-global. + NULL input is preserved (NULL → NULL). + Applicable to: string, int, long, binary """ @@ -1663,10 +1675,28 @@ class AddSchemaUpdate(BaseUpdate): class ReadRestrictions(BaseModel): """ Read restrictions for a table, including column projections and row filter expressions. - A client MUST enforce the restrictions defined in this object when reading data from the table. + A reader MUST enforce the restrictions defined in this object when reading data from the table. Read restrictions returned in a loadTable response apply to every read operation on the loaded table performed using this response, including subsequent planTableScan and fetchScanTasks calls. + In this section, "reader" refers to the read-side actor that applies restrictions per row or per column. "Engine" refers to the broader query-execution context that defines query lifetime and scope (e.g. a SQL session, a single PyIceberg scan), and is the actor responsible for query-scoped behavior such as salt generation in sha-256-query-local. These restrictions apply only to the authenticated principal, user, or account associated with the request. They MUST NOT be interpreted as global policy and MUST NOT be applied beyond the entity identified by the Authentication header (or other applicable authentication mechanism). - If both properties are absent or empty, the ReadRestrictions object imposes no restrictions and is equivalent to the field being absent from the response. A server MUST NOT return an action for a column whose type is not listed in that action's "Applicable to" set. For all actions, if the input column value is NULL, the output MUST be NULL. + An empty ReadRestrictions object (no required-column-projections and no required-row-filter) imposes no restrictions and is equivalent to the field being absent from the response. A server MUST NOT return an action for a column whose type is not listed in that action's "Applicable to" set. + NULL handling is action-specific. Each action's description specifies its behavior on NULL input. If a column projection targets a struct-typed field, other column projections in the same ReadRestrictions MUST NOT target any of that struct's subfields (at any depth). This avoids ambiguity about which action governs a given leaf value. + Example: + + { + "required-column-projections": [ + { "field-id": 4, "action": "show-last-4" }, + { "field-id": 6, "action": "replace-with-null" }, + { "field-id": 8, "action": "truncate-to-year" }, + { "field-id": 10, "action": "sha-256-global" }, + { "field-id": 12, "action": "mask-alphanum" } + ], + "required-row-filter": { + "type": "eq", + "term": "region", + "value": "US" + } + } """ @@ -1681,32 +1711,20 @@ class ReadRestrictions(BaseModel): | TruncateToMonth | Sha256Global | Sha256QueryLocal - | ApplyExpression ] | None ) = Field( None, alias='required-column-projections', - description="A list of columns that require specific actions to be applied when reading.\nIf this property is absent, a reader MAY access all columns of the table as-is without any mandatory transformations.\nIf this property is present, each listed column MUST have its specified action applied. Columns not listed in required-column-projections are not subject to any read restrictions.\nWhen this list is present:\n1. For each column listed in required-column-projections, the reader MUST apply\n the specified action before returning values for that column.\n\n2. The reader MUST replace all output references to the column with the result\n of the action, presenting the result under the original column name. For\n example, if the action for column cc is mask-alphanum, the reader MUST\n return the masked value as cc in the query output.\n\n3. Columns not listed in required-column-projections MAY be projected normally\n by the reader without any mandatory transformations.\n\n4. A column MUST appear at most once in required-column-projections.\n5. If a projected column's action cannot be evaluated by the reader\n (including unrecognized action types), the reader MUST fail rather than\n ignore or skip the action.\n\n6. Each action defines the output type for its column. For all predefined\n actions except apply-expression, the output type matches the input column\n type. For apply-expression, the output type is determined by the expression.\n", + description="A list of columns that require specific actions to be applied when reading.\nIf this property is absent, a reader MAY access all columns of the table as-is without any mandatory transformations.\nIf this property is present, each listed column MUST have its specified action applied. Columns not listed in required-column-projections are not subject to any read restrictions.\nWhen this list is present:\n1. For each column listed in required-column-projections, the reader MUST apply\n the specified action before returning values for that column.\n\n2. The reader MUST replace all output references to the column with the result\n of the action, presenting the result under the original column name. For\n example, if the action for column cc is mask-alphanum, the reader MUST\n return the masked value as cc in the query output.\n\n3. Columns not listed in required-column-projections MAY be projected normally\n by the reader without any mandatory transformations.\n\n4. A column MUST appear at most once in required-column-projections.\n5. If a projected column's action cannot be evaluated by the reader (including\n unrecognized action types), the reader MUST fail the query with an error to\n the caller. The reader MUST NOT silently return raw, partial, or empty\n results to mask the failure.\n\n6. Each action defines the output type for its column. For all predefined\n actions, the output type matches the input column type.\n", ) required_row_filter: Expression | None = Field( None, alias='required-row-filter', - description='An expression that filters rows in the table that the authenticated principal does not have access to.\n1. The expression MUST evaluate to a boolean. A reader MUST discard any row for which\n the filter evaluates to FALSE, and no information derived from discarded rows\n MAY be included in the query result.\n\n2. Row filters MUST be evaluated against the original, untransformed column values.\n Required projections MUST be applied only after row filters are applied.\n\n3. If a client cannot interpret or evaluate a provided filter expression, it MUST fail.\n4. If this property is absent, null, or always true then no mandatory filtering is required.\n', + description='An expression that filters rows in the table that the authenticated principal does not have access to.\n1. The expression MUST evaluate to a boolean (TRUE or FALSE; Iceberg expressions\n never produce NULL). A reader MUST discard any row for which\n the filter evaluates to FALSE, and no information derived from discarded rows\n MAY be included in the query result.\n\n2. Row filters MUST be evaluated against the original, untransformed column values.\n Required projections MUST be applied only after row filters are applied.\n\n3. If a reader cannot interpret or evaluate a provided filter expression, it MUST\n fail the query with an error to the caller. The reader MUST NOT silently return\n partial, raw, or empty results to mask the failure.\n\n4. If this property is absent, null, or always true then no mandatory filtering is required.\n', ) -class ApplyExpression(Action): - """ - Replace the field with the result of an expression. Produce the original field name with the expression result. - Applicable to: all data types - - """ - - action: Literal['apply-expression'] - expression: Expression - - class LoadTableResult(BaseModel): """ Result used when a table is successfully loaded. @@ -2272,7 +2290,6 @@ class PlanTableScanResult( ViewMetadata.model_rebuild() AddSchemaUpdate.model_rebuild() ReadRestrictions.model_rebuild() -ApplyExpression.model_rebuild() ScanTasks.model_rebuild() CommitTableRequest.model_rebuild() CommitViewRequest.model_rebuild() diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index 1714116c4a90..dabbda639fc9 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3668,24 +3668,52 @@ components: description: > Read restrictions for a table, including column projections and row filter expressions. - A client MUST enforce the restrictions defined in this object when reading data - from the table. + A reader MUST enforce the restrictions defined in this object when reading data + from the table. Read restrictions returned in a loadTable response apply to every + read operation on the loaded table performed using this response, including + subsequent planTableScan and fetchScanTasks calls. + + In this section, "reader" refers to the read-side actor that applies restrictions + per row or per column. "Engine" refers to the broader query-execution context + that defines query lifetime and scope (e.g. a SQL session, a single PyIceberg + scan), and is the actor responsible for query-scoped behavior such as salt + generation in sha-256-query-local. These restrictions apply only to the authenticated principal, user, or account associated with the request. They MUST NOT be interpreted as global policy and MUST NOT be applied beyond the entity identified by the Authentication header (or other applicable authentication mechanism). - If both properties are absent or empty, the ReadRestrictions object imposes no - restrictions and is equivalent to the field being absent from the response. + An empty ReadRestrictions object (no required-column-projections and no + required-row-filter) imposes no restrictions and is equivalent to the field + being absent from the response. A server MUST NOT return an action for a column whose type is not listed in that action's "Applicable to" set. - For all actions, if the input column value is NULL, the output MUST be NULL. + + NULL handling is action-specific. Each action's description specifies its + behavior on NULL input. If a column projection targets a struct-typed field, other column projections in the same ReadRestrictions MUST NOT target any of that struct's subfields (at any depth). This avoids ambiguity about which action governs a given leaf value. + + Example: + + { + "required-column-projections": [ + { "field-id": 4, "action": "show-last-4" }, + { "field-id": 6, "action": "replace-with-null" }, + { "field-id": 8, "action": "truncate-to-year" }, + { "field-id": 10, "action": "sha-256-global" }, + { "field-id": 12, "action": "mask-alphanum" } + ], + "required-row-filter": { + "type": "eq", + "term": "region", + "value": "US" + } + } properties: required-column-projections: description: > @@ -3713,13 +3741,13 @@ components: 4. A column MUST appear at most once in required-column-projections. - 5. If a projected column's action cannot be evaluated by the reader - (including unrecognized action types), the reader MUST fail rather than - ignore or skip the action. + 5. If a projected column's action cannot be evaluated by the reader (including + unrecognized action types), the reader MUST fail the query with an error to + the caller. The reader MUST NOT silently return raw, partial, or empty + results to mask the failure. 6. Each action defines the output type for its column. For all predefined - actions except apply-expression, the output type matches the input column - type. For apply-expression, the output type is determined by the expression. + actions, the output type matches the input column type. type: array items: @@ -3728,17 +3756,21 @@ components: description: > An expression that filters rows in the table that the authenticated principal does not have access to. - 1. The expression MUST evaluate to a boolean. A reader MUST discard any row for which + 1. The expression MUST evaluate to a boolean (TRUE or FALSE; Iceberg expressions + never produce NULL). A reader MUST discard any row for which the filter evaluates to FALSE, and no information derived from discarded rows MAY be included in the query result. 2. Row filters MUST be evaluated against the original, untransformed column values. Required projections MUST be applied only after row filters are applied. - 3. If a client cannot interpret or evaluate a provided filter expression, it MUST fail. + 3. If a reader cannot interpret or evaluate a provided filter expression, it MUST + fail the query with an error to the caller. The reader MUST NOT silently return + partial, raw, or empty results to mask the failure. 4. If this property is absent, null, or always true then no mandatory filtering is required. - $ref: '#/components/schemas/Expression' + allOf: + - $ref: '#/components/schemas/Expression' Action: discriminator: @@ -3753,7 +3785,6 @@ components: truncate-to-month: '#/components/schemas/TruncateToMonth' sha-256-global: '#/components/schemas/Sha256Global' sha-256-query-local: '#/components/schemas/Sha256QueryLocal' - apply-expression: '#/components/schemas/ApplyExpression' type: object required: - action @@ -3763,7 +3794,7 @@ components: type: string field-id: type: integer - description: field id of the column being projected. + description: Field ID of the column being projected. MaskAlphanum: description: > @@ -3782,6 +3813,8 @@ components: For example: "prashant010696@gmail.com" → "xxxxxxxxnnnnnn@xxxxx.xxx" + NULL input is preserved (NULL → NULL). + Applicable to: string allOf: - $ref: '#/components/schemas/Action' @@ -3793,7 +3826,7 @@ components: MaskToFixedValue: description: > Replaces the column value with a predefined type-specific fixed value. - Engines MUST use exactly the values listed below to ensure consistency + Readers MUST use exactly the values listed below to ensure consistency across implementations. Fixed values by type: @@ -3820,6 +3853,8 @@ components: - map: empty map {} - struct: struct with each field set to its type-specific default (applied recursively) + NULL input is also replaced with the type-specific fixed value; NULL is not preserved. + Applicable to: all data types allOf: - $ref: '#/components/schemas/Action' @@ -3830,7 +3865,8 @@ components: ReplaceWithNull: description: > - Replaces the entire column value with NULL. + Replaces the entire column value with NULL. NULL input is preserved (NULL → NULL). + A server MUST NOT return this action for a non-nullable (required) column. Applicable to: all nullable types allOf: @@ -3848,6 +3884,8 @@ components: For example: "prashant010696@gmail.com" → "prasxxxxnnnnnn@xxxxx.xxx" + NULL input is preserved (NULL → NULL). + Applicable to: string allOf: - $ref: '#/components/schemas/Action' @@ -3864,6 +3902,8 @@ components: For example: "4111-1111-1111-4444" → "nnnn-nnnn-nnnn-4444" + NULL input is preserved (NULL → NULL). + Applicable to: string allOf: - $ref: '#/components/schemas/Action' @@ -3880,6 +3920,8 @@ components: For example: 2024-07-15 → 2024-01-01 For timestamptz and timestamptz_ns, truncation is performed in UTC. + NULL input is preserved (NULL → NULL). + Applicable to: date, timestamp, timestamptz, timestamp_ns, timestamptz_ns allOf: - $ref: '#/components/schemas/Action' @@ -3896,6 +3938,8 @@ components: For example: 2024-07-15 → 2024-07-01 For timestamptz and timestamptz_ns, truncation is performed in UTC. + NULL input is preserved (NULL → NULL). + Applicable to: date, timestamp, timestamptz, timestamp_ns, timestamptz_ns allOf: - $ref: '#/components/schemas/Action' @@ -3921,6 +3965,8 @@ components: - long: first 8 bytes of the digest, read as a signed two's complement little-endian long - binary: the full 32-byte raw SHA-256 digest + NULL input is preserved (NULL → NULL). + Applicable to: string, int, long, binary allOf: - $ref: '#/components/schemas/Action' @@ -3934,12 +3980,16 @@ components: Applies SHA-256 with a per-query random salt, making the output non-deterministic across queries while remaining consistent within a single query. - The engine MUST generate a cryptographically random salt of at least 16 bytes for each query and apply it as: - SHA-256(salt_bytes || canonical_bytes) - where canonical_bytes follows the same encoding rules as sha-256-global. + The engine MUST generate a cryptographically random salt of at least 16 bytes for each query. + + For each column value, the engine MUST encode the value to bytes using + sha-256-global's input rules, prepend the per-query salt, and compute + SHA-256 over the result. Output encoding follows the same rules as sha-256-global. + NULL input is preserved (NULL → NULL). + Applicable to: string, int, long, binary allOf: - $ref: '#/components/schemas/Action' @@ -3948,24 +3998,6 @@ components: type: string const: "sha-256-query-local" - ApplyExpression: - description: > - Replace the field with the result of an expression. Produce the original field name - with the expression result. - - Applicable to: all data types - allOf: - - $ref: '#/components/schemas/Action' - required: - - action - - expression - properties: - action: - type: string - const: "apply-expression" - expression: - $ref: '#/components/schemas/Expression' - LoadCredentialsResponse: type: object required: From ed944a3c35dd39e271b7d945cc195fe46ee46151 Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Tue, 9 Jun 2026 14:34:46 -0700 Subject: [PATCH 25/45] Spec: V1 conditional wording for read-restrictions Replace unconditional MUST with SHOULD support + conditional MUST fail-closed. Clients that support read-restrictions MUST fail if they cannot apply any returned restriction (including unrecognized action or expression types). This leaves room for V2 to tighten to unconditional. --- open-api/rest-catalog-open-api.yaml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index dabbda639fc9..a3579ea1a8ad 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3668,10 +3668,12 @@ components: description: > Read restrictions for a table, including column projections and row filter expressions. - A reader MUST enforce the restrictions defined in this object when reading data - from the table. Read restrictions returned in a loadTable response apply to every - read operation on the loaded table performed using this response, including - subsequent planTableScan and fetchScanTasks calls. + A client SHOULD support the read-restrictions field. If a client supports + read-restrictions, it MUST fail if it cannot apply any returned restriction + (including unrecognized action or expression types). Read restrictions returned + in a loadTable response apply to every read operation on the loaded table + performed using this response, including subsequent planTableScan and + fetchScanTasks calls. In this section, "reader" refers to the read-side actor that applies restrictions per row or per column. "Engine" refers to the broader query-execution context From 2e91958eb626cb7357c806a0405ddf3509880b6f Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Tue, 9 Jun 2026 14:35:13 -0700 Subject: [PATCH 26/45] Spec: Update Python model with V1 read-restrictions wording --- open-api/rest-catalog-open-api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index 43d327c5a6ef..55f514731e95 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -1675,7 +1675,7 @@ class AddSchemaUpdate(BaseUpdate): class ReadRestrictions(BaseModel): """ Read restrictions for a table, including column projections and row filter expressions. - A reader MUST enforce the restrictions defined in this object when reading data from the table. Read restrictions returned in a loadTable response apply to every read operation on the loaded table performed using this response, including subsequent planTableScan and fetchScanTasks calls. + A client SHOULD support the read-restrictions field. If a client supports read-restrictions, it MUST fail if it cannot apply any returned restriction (including unrecognized action or expression types). Read restrictions returned in a loadTable response apply to every read operation on the loaded table performed using this response, including subsequent planTableScan and fetchScanTasks calls. In this section, "reader" refers to the read-side actor that applies restrictions per row or per column. "Engine" refers to the broader query-execution context that defines query lifetime and scope (e.g. a SQL session, a single PyIceberg scan), and is the actor responsible for query-scoped behavior such as salt generation in sha-256-query-local. These restrictions apply only to the authenticated principal, user, or account associated with the request. They MUST NOT be interpreted as global policy and MUST NOT be applied beyond the entity identified by the Authentication header (or other applicable authentication mechanism). An empty ReadRestrictions object (no required-column-projections and no required-row-filter) imposes no restrictions and is equivalent to the field being absent from the response. A server MUST NOT return an action for a column whose type is not listed in that action's "Applicable to" set. From 3dc5a79e6eea70439cc3e09278379b71918d68f3 Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Sun, 14 Jun 2026 15:41:40 -0700 Subject: [PATCH 27/45] Spec: Lowercase RFC2119 keywords and drop redundant MAYs Address rdblue review on PR #13879: - L3635: lowercase MUST/SHOULD/MAY across the ReadRestrictions section. - L3711: drop the duplicated "MAY be projected normally" rule from required-column-projections; renumber rules 4-6 to 3-5. - L3698 (partial): drop the "If absent, a reader MAY access all columns" sentence; absence is implied by the columns-not-listed semantics. --- open-api/rest-catalog-open-api.py | 20 +++++----- open-api/rest-catalog-open-api.yaml | 58 +++++++++++++---------------- 2 files changed, 36 insertions(+), 42 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index 55f514731e95..aa67cf2ad01d 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -619,7 +619,7 @@ class MaskAlphanum(Action): class MaskToFixedValue(Action): """ - Replaces the column value with a predefined type-specific fixed value. Readers MUST use exactly the values listed below to ensure consistency across implementations. + Replaces the column value with a predefined type-specific fixed value. Readers must use exactly the values listed below to ensure consistency across implementations. Fixed values by type: - boolean: false - int: 0 - long: 0 - float: 0.0 - double: 0.0 - decimal(p, s): 0 (zero with s digits after the decimal point, e.g. 0.00 for decimal(p,2)) - string: "XXXXXXXX" - date: 1970-01-01 - time: 00:00:00 - timestamp: 1970-01-01T00:00:00 - timestamptz: 1970-01-01T00:00:00+00:00 - timestamp_ns: 1970-01-01T00:00:00.000000000 - timestamptz_ns: 1970-01-01T00:00:00.000000000+00:00 - uuid: 00000000-0000-0000-0000-000000000000 - fixed(n): n zero bytes - binary: empty byte sequence - variant: {} - geometry: POINT EMPTY - geography: POINT EMPTY - list: empty list [] - map: empty map {} - struct: struct with each field set to its type-specific default (applied recursively) NULL input is also replaced with the type-specific fixed value; NULL is not preserved. Applicable to: all data types @@ -631,7 +631,7 @@ class MaskToFixedValue(Action): class ReplaceWithNull(Action): """ - Replaces the entire column value with NULL. NULL input is preserved (NULL → NULL). A server MUST NOT return this action for a non-nullable (required) column. + Replaces the entire column value with NULL. NULL input is preserved (NULL → NULL). A server must not return this action for a non-nullable (required) column. Applicable to: all nullable types """ @@ -718,9 +718,9 @@ class Sha256QueryLocal(Action): Applies SHA-256 with a per-query random salt, making the output non-deterministic across queries while remaining consistent within a single query. - The engine MUST generate a cryptographically random salt of at least 16 bytes for each query. + The engine must generate a cryptographically random salt of at least 16 bytes for each query. - For each column value, the engine MUST encode the value to bytes using + For each column value, the engine must encode the value to bytes using sha-256-global's input rules, prepend the per-query salt, and compute SHA-256 over the result. @@ -1675,12 +1675,12 @@ class AddSchemaUpdate(BaseUpdate): class ReadRestrictions(BaseModel): """ Read restrictions for a table, including column projections and row filter expressions. - A client SHOULD support the read-restrictions field. If a client supports read-restrictions, it MUST fail if it cannot apply any returned restriction (including unrecognized action or expression types). Read restrictions returned in a loadTable response apply to every read operation on the loaded table performed using this response, including subsequent planTableScan and fetchScanTasks calls. + A client should support the read-restrictions field. If a client supports read-restrictions, it must fail if it cannot apply any returned restriction (including unrecognized action or expression types). Read restrictions returned in a loadTable response apply to every read operation on the loaded table performed using this response, including subsequent planTableScan and fetchScanTasks calls. In this section, "reader" refers to the read-side actor that applies restrictions per row or per column. "Engine" refers to the broader query-execution context that defines query lifetime and scope (e.g. a SQL session, a single PyIceberg scan), and is the actor responsible for query-scoped behavior such as salt generation in sha-256-query-local. - These restrictions apply only to the authenticated principal, user, or account associated with the request. They MUST NOT be interpreted as global policy and MUST NOT be applied beyond the entity identified by the Authentication header (or other applicable authentication mechanism). - An empty ReadRestrictions object (no required-column-projections and no required-row-filter) imposes no restrictions and is equivalent to the field being absent from the response. A server MUST NOT return an action for a column whose type is not listed in that action's "Applicable to" set. + These restrictions apply only to the authenticated principal, user, or account associated with the request. They must not be interpreted as global policy and must not be applied beyond the entity identified by the Authentication header (or other applicable authentication mechanism). + An empty ReadRestrictions object (no required-column-projections and no required-row-filter) imposes no restrictions and is equivalent to the field being absent from the response. A server must not return an action for a column whose type is not listed in that action's "Applicable to" set. NULL handling is action-specific. Each action's description specifies its behavior on NULL input. - If a column projection targets a struct-typed field, other column projections in the same ReadRestrictions MUST NOT target any of that struct's subfields (at any depth). This avoids ambiguity about which action governs a given leaf value. + If a column projection targets a struct-typed field, other column projections in the same ReadRestrictions must not target any of that struct's subfields (at any depth). This avoids ambiguity about which action governs a given leaf value. Example: { @@ -1716,12 +1716,12 @@ class ReadRestrictions(BaseModel): ) = Field( None, alias='required-column-projections', - description="A list of columns that require specific actions to be applied when reading.\nIf this property is absent, a reader MAY access all columns of the table as-is without any mandatory transformations.\nIf this property is present, each listed column MUST have its specified action applied. Columns not listed in required-column-projections are not subject to any read restrictions.\nWhen this list is present:\n1. For each column listed in required-column-projections, the reader MUST apply\n the specified action before returning values for that column.\n\n2. The reader MUST replace all output references to the column with the result\n of the action, presenting the result under the original column name. For\n example, if the action for column cc is mask-alphanum, the reader MUST\n return the masked value as cc in the query output.\n\n3. Columns not listed in required-column-projections MAY be projected normally\n by the reader without any mandatory transformations.\n\n4. A column MUST appear at most once in required-column-projections.\n5. If a projected column's action cannot be evaluated by the reader (including\n unrecognized action types), the reader MUST fail the query with an error to\n the caller. The reader MUST NOT silently return raw, partial, or empty\n results to mask the failure.\n\n6. Each action defines the output type for its column. For all predefined\n actions, the output type matches the input column type.\n", + description="A list of columns that require specific actions to be applied when reading.\nIf this property is present, each listed column must have its specified action applied. Columns not listed in required-column-projections are not subject to any read restrictions.\nWhen this list is present:\n1. For each column listed in required-column-projections, the reader must apply\n the specified action before returning values for that column.\n\n2. The reader must replace all output references to the column with the result\n of the action, presenting the result under the original column name. For\n example, if the action for column cc is mask-alphanum, the reader must\n return the masked value as cc in the query output.\n\n3. A column must appear at most once in required-column-projections.\n4. If a projected column's action cannot be evaluated by the reader (including\n unrecognized action types), the reader must fail the query with an error to\n the caller. The reader must not silently return raw, partial, or empty\n results to mask the failure.\n\n5. Each action defines the output type for its column. For all predefined\n actions, the output type matches the input column type.\n", ) required_row_filter: Expression | None = Field( None, alias='required-row-filter', - description='An expression that filters rows in the table that the authenticated principal does not have access to.\n1. The expression MUST evaluate to a boolean (TRUE or FALSE; Iceberg expressions\n never produce NULL). A reader MUST discard any row for which\n the filter evaluates to FALSE, and no information derived from discarded rows\n MAY be included in the query result.\n\n2. Row filters MUST be evaluated against the original, untransformed column values.\n Required projections MUST be applied only after row filters are applied.\n\n3. If a reader cannot interpret or evaluate a provided filter expression, it MUST\n fail the query with an error to the caller. The reader MUST NOT silently return\n partial, raw, or empty results to mask the failure.\n\n4. If this property is absent, null, or always true then no mandatory filtering is required.\n', + description='An expression that filters rows in the table that the authenticated principal does not have access to.\n1. The expression must evaluate to a boolean (TRUE or FALSE; Iceberg expressions\n never produce NULL). A reader must discard any row for which\n the filter evaluates to FALSE, and no information derived from discarded rows\n may be included in the query result.\n\n2. Row filters must be evaluated against the original, untransformed column values.\n Required projections must be applied only after row filters are applied.\n\n3. If a reader cannot interpret or evaluate a provided filter expression, it must\n fail the query with an error to the caller. The reader must not silently return\n partial, raw, or empty results to mask the failure.\n\n4. If this property is absent, null, or always true then no mandatory filtering is required.\n', ) diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index a3579ea1a8ad..1142b8b5b018 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3668,8 +3668,8 @@ components: description: > Read restrictions for a table, including column projections and row filter expressions. - A client SHOULD support the read-restrictions field. If a client supports - read-restrictions, it MUST fail if it cannot apply any returned restriction + A client should support the read-restrictions field. If a client supports + read-restrictions, it must fail if it cannot apply any returned restriction (including unrecognized action or expression types). Read restrictions returned in a loadTable response apply to every read operation on the loaded table performed using this response, including subsequent planTableScan and @@ -3682,21 +3682,21 @@ components: generation in sha-256-query-local. These restrictions apply only to the authenticated principal, user, or account - associated with the request. They MUST NOT be interpreted as global policy and - MUST NOT be applied beyond the entity identified by the Authentication header + associated with the request. They must not be interpreted as global policy and + must not be applied beyond the entity identified by the Authentication header (or other applicable authentication mechanism). An empty ReadRestrictions object (no required-column-projections and no required-row-filter) imposes no restrictions and is equivalent to the field being absent from the response. - A server MUST NOT return an action for a column whose type is not listed in + A server must not return an action for a column whose type is not listed in that action's "Applicable to" set. NULL handling is action-specific. Each action's description specifies its behavior on NULL input. If a column projection targets a struct-typed field, other column projections - in the same ReadRestrictions MUST NOT target any of that struct's subfields + in the same ReadRestrictions must not target any of that struct's subfields (at any depth). This avoids ambiguity about which action governs a given leaf value. @@ -3721,34 +3721,28 @@ components: description: > A list of columns that require specific actions to be applied when reading. - If this property is absent, a reader MAY access all columns of the table as-is - without any mandatory transformations. - - If this property is present, each listed column MUST have its specified + If this property is present, each listed column must have its specified action applied. Columns not listed in required-column-projections are not subject to any read restrictions. When this list is present: - 1. For each column listed in required-column-projections, the reader MUST apply + 1. For each column listed in required-column-projections, the reader must apply the specified action before returning values for that column. - 2. The reader MUST replace all output references to the column with the result + 2. The reader must replace all output references to the column with the result of the action, presenting the result under the original column name. For - example, if the action for column cc is mask-alphanum, the reader MUST + example, if the action for column cc is mask-alphanum, the reader must return the masked value as cc in the query output. - 3. Columns not listed in required-column-projections MAY be projected normally - by the reader without any mandatory transformations. - - 4. A column MUST appear at most once in required-column-projections. + 3. A column must appear at most once in required-column-projections. - 5. If a projected column's action cannot be evaluated by the reader (including - unrecognized action types), the reader MUST fail the query with an error to - the caller. The reader MUST NOT silently return raw, partial, or empty + 4. If a projected column's action cannot be evaluated by the reader (including + unrecognized action types), the reader must fail the query with an error to + the caller. The reader must not silently return raw, partial, or empty results to mask the failure. - 6. Each action defines the output type for its column. For all predefined + 5. Each action defines the output type for its column. For all predefined actions, the output type matches the input column type. type: array @@ -3758,16 +3752,16 @@ components: description: > An expression that filters rows in the table that the authenticated principal does not have access to. - 1. The expression MUST evaluate to a boolean (TRUE or FALSE; Iceberg expressions - never produce NULL). A reader MUST discard any row for which + 1. The expression must evaluate to a boolean (TRUE or FALSE; Iceberg expressions + never produce NULL). A reader must discard any row for which the filter evaluates to FALSE, and no information derived from discarded rows - MAY be included in the query result. + may be included in the query result. - 2. Row filters MUST be evaluated against the original, untransformed column values. - Required projections MUST be applied only after row filters are applied. + 2. Row filters must be evaluated against the original, untransformed column values. + Required projections must be applied only after row filters are applied. - 3. If a reader cannot interpret or evaluate a provided filter expression, it MUST - fail the query with an error to the caller. The reader MUST NOT silently return + 3. If a reader cannot interpret or evaluate a provided filter expression, it must + fail the query with an error to the caller. The reader must not silently return partial, raw, or empty results to mask the failure. 4. If this property is absent, null, or always true then no mandatory filtering is required. @@ -3828,7 +3822,7 @@ components: MaskToFixedValue: description: > Replaces the column value with a predefined type-specific fixed value. - Readers MUST use exactly the values listed below to ensure consistency + Readers must use exactly the values listed below to ensure consistency across implementations. Fixed values by type: @@ -3868,7 +3862,7 @@ components: ReplaceWithNull: description: > Replaces the entire column value with NULL. NULL input is preserved (NULL → NULL). - A server MUST NOT return this action for a non-nullable (required) column. + A server must not return this action for a non-nullable (required) column. Applicable to: all nullable types allOf: @@ -3982,9 +3976,9 @@ components: Applies SHA-256 with a per-query random salt, making the output non-deterministic across queries while remaining consistent within a single query. - The engine MUST generate a cryptographically random salt of at least 16 bytes for each query. + The engine must generate a cryptographically random salt of at least 16 bytes for each query. - For each column value, the engine MUST encode the value to bytes using + For each column value, the engine must encode the value to bytes using sha-256-global's input rules, prepend the per-query salt, and compute SHA-256 over the result. From a4dcd0bd7f6818a28e076504d15c970d382c9462 Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Sun, 14 Jun 2026 15:54:36 -0700 Subject: [PATCH 28/45] Spec: Restructure ReadRestrictions per rdblue review - Move client-support, response-scope, and principal-scoping prose to loadTable.description (L3640, L3645, L3656). - Delete the reader/engine glossary paragraph; engine is self-defined in sha-256-query-local (L3651). - Move the type-applicability rule into required-column-projections (L3661). - Replace the struct-subfield rule with an outermost-wins precedence rule that covers struct, list, and map containers (L3665, L3670). - Replace the inline JSON example with a yaml example: block (L3672). - Move the type invariant, evaluation order, and reader failure semantics up to the parent description (L3721, L3736, L3740). --- open-api/rest-catalog-open-api.py | 31 ++------- open-api/rest-catalog-open-api.yaml | 101 ++++++++++++---------------- 2 files changed, 48 insertions(+), 84 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index aa67cf2ad01d..4639f3ed49e2 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -1674,29 +1674,10 @@ class AddSchemaUpdate(BaseUpdate): class ReadRestrictions(BaseModel): """ - Read restrictions for a table, including column projections and row filter expressions. - A client should support the read-restrictions field. If a client supports read-restrictions, it must fail if it cannot apply any returned restriction (including unrecognized action or expression types). Read restrictions returned in a loadTable response apply to every read operation on the loaded table performed using this response, including subsequent planTableScan and fetchScanTasks calls. - In this section, "reader" refers to the read-side actor that applies restrictions per row or per column. "Engine" refers to the broader query-execution context that defines query lifetime and scope (e.g. a SQL session, a single PyIceberg scan), and is the actor responsible for query-scoped behavior such as salt generation in sha-256-query-local. - These restrictions apply only to the authenticated principal, user, or account associated with the request. They must not be interpreted as global policy and must not be applied beyond the entity identified by the Authentication header (or other applicable authentication mechanism). - An empty ReadRestrictions object (no required-column-projections and no required-row-filter) imposes no restrictions and is equivalent to the field being absent from the response. A server must not return an action for a column whose type is not listed in that action's "Applicable to" set. - NULL handling is action-specific. Each action's description specifies its behavior on NULL input. - If a column projection targets a struct-typed field, other column projections in the same ReadRestrictions must not target any of that struct's subfields (at any depth). This avoids ambiguity about which action governs a given leaf value. - Example: - - { - "required-column-projections": [ - { "field-id": 4, "action": "show-last-4" }, - { "field-id": 6, "action": "replace-with-null" }, - { "field-id": 8, "action": "truncate-to-year" }, - { "field-id": 10, "action": "sha-256-global" }, - { "field-id": 12, "action": "mask-alphanum" } - ], - "required-row-filter": { - "type": "eq", - "term": "region", - "value": "US" - } - } + Read restrictions for a table. + A reader evaluates the row filter against original, untransformed column values, then applies required-column-projections to the surviving rows. Each action must produce a value of the same type as the input column. If a reader cannot apply any returned restriction (a filter expression or an action), it must fail the query and must not silently return raw, partial, or empty results. + If a projection targets a field, that action governs every value reachable through it; other projections in the same object that target a descendant of that field have no effect. + An empty ReadRestrictions object (no required-column-projections and no required-row-filter) imposes no restrictions and is equivalent to the field being absent from the response. """ @@ -1716,12 +1697,12 @@ class ReadRestrictions(BaseModel): ) = Field( None, alias='required-column-projections', - description="A list of columns that require specific actions to be applied when reading.\nIf this property is present, each listed column must have its specified action applied. Columns not listed in required-column-projections are not subject to any read restrictions.\nWhen this list is present:\n1. For each column listed in required-column-projections, the reader must apply\n the specified action before returning values for that column.\n\n2. The reader must replace all output references to the column with the result\n of the action, presenting the result under the original column name. For\n example, if the action for column cc is mask-alphanum, the reader must\n return the masked value as cc in the query output.\n\n3. A column must appear at most once in required-column-projections.\n4. If a projected column's action cannot be evaluated by the reader (including\n unrecognized action types), the reader must fail the query with an error to\n the caller. The reader must not silently return raw, partial, or empty\n results to mask the failure.\n\n5. Each action defines the output type for its column. For all predefined\n actions, the output type matches the input column type.\n", + description='A list of columns that require specific actions to be applied when reading. A server must not return an action for a column whose type is not listed in that action\'s "Applicable to" set.\nIf this property is present, each listed column must have its specified action applied. Columns not listed in required-column-projections are not subject to any read restrictions.\nWhen this list is present:\n1. For each column listed in required-column-projections, the reader must apply\n the specified action before returning values for that column.\n\n2. The reader must replace all output references to the column with the result\n of the action, presenting the result under the original column name. For\n example, if the action for column cc is mask-alphanum, the reader must\n return the masked value as cc in the query output.\n\n3. A column must appear at most once in required-column-projections.\n4. If a projected column\'s action cannot be evaluated by the reader (including\n unrecognized action types), the reader must fail the query with an error to\n the caller. The reader must not silently return raw, partial, or empty\n results to mask the failure.\n', ) required_row_filter: Expression | None = Field( None, alias='required-row-filter', - description='An expression that filters rows in the table that the authenticated principal does not have access to.\n1. The expression must evaluate to a boolean (TRUE or FALSE; Iceberg expressions\n never produce NULL). A reader must discard any row for which\n the filter evaluates to FALSE, and no information derived from discarded rows\n may be included in the query result.\n\n2. Row filters must be evaluated against the original, untransformed column values.\n Required projections must be applied only after row filters are applied.\n\n3. If a reader cannot interpret or evaluate a provided filter expression, it must\n fail the query with an error to the caller. The reader must not silently return\n partial, raw, or empty results to mask the failure.\n\n4. If this property is absent, null, or always true then no mandatory filtering is required.\n', + description='An expression that filters rows in the table that the authenticated principal does not have access to.\n1. The expression must evaluate to a boolean (TRUE or FALSE; Iceberg expressions\n never produce NULL). A reader must discard any row for which\n the filter evaluates to FALSE, and no information derived from discarded rows\n may be included in the query result.\n\n2. If this property is absent, null, or always true then no mandatory filtering is required.\n', ) diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index 1142b8b5b018..4a279c34d16f 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -1051,6 +1051,15 @@ paths: table. The configuration key "token" is used to pass an access token to be used as a bearer token for table requests. Otherwise, a token may be passed using a RFC 8693 token type as a configuration key. For example, "urn:ietf:params:oauth:token-type:jwt=". + + + The response may include a read-restrictions field. A client should support this field; + if it does, it must fail any read against the loaded table when it cannot apply a returned + restriction. This includes unrecognized action or expression types, and actions whose + definition cannot be loaded, parsed, or evaluated. These restrictions apply to every read + performed using this response (including subsequent planTableScan and fetchScanTasks + calls), only to the authenticated principal associated with the request, and must not be + interpreted as global policy. parameters: - $ref: '#/components/parameters/data-access' - name: If-None-Match @@ -3666,60 +3675,44 @@ components: ReadRestrictions: type: object description: > - Read restrictions for a table, including column projections and row filter expressions. - - A client should support the read-restrictions field. If a client supports - read-restrictions, it must fail if it cannot apply any returned restriction - (including unrecognized action or expression types). Read restrictions returned - in a loadTable response apply to every read operation on the loaded table - performed using this response, including subsequent planTableScan and - fetchScanTasks calls. - - In this section, "reader" refers to the read-side actor that applies restrictions - per row or per column. "Engine" refers to the broader query-execution context - that defines query lifetime and scope (e.g. a SQL session, a single PyIceberg - scan), and is the actor responsible for query-scoped behavior such as salt - generation in sha-256-query-local. - - These restrictions apply only to the authenticated principal, user, or account - associated with the request. They must not be interpreted as global policy and - must not be applied beyond the entity identified by the Authentication header - (or other applicable authentication mechanism). + Read restrictions for a table. + + A reader evaluates the row filter against original, untransformed column + values, then applies required-column-projections to the surviving rows. + Each action must produce a value of the same type as the input column. + If a reader cannot apply any returned restriction (a filter expression + or an action), it must fail the query and must not silently return raw, + partial, or empty results. + + If a projection targets a field, that action governs every value reachable + through it; other projections in the same object that target a descendant + of that field have no effect. An empty ReadRestrictions object (no required-column-projections and no - required-row-filter) imposes no restrictions and is equivalent to the field - being absent from the response. - A server must not return an action for a column whose type is not listed in - that action's "Applicable to" set. - - NULL handling is action-specific. Each action's description specifies its - behavior on NULL input. - - If a column projection targets a struct-typed field, other column projections - in the same ReadRestrictions must not target any of that struct's subfields - (at any depth). This avoids ambiguity about which action governs a given - leaf value. - - Example: - - { - "required-column-projections": [ - { "field-id": 4, "action": "show-last-4" }, - { "field-id": 6, "action": "replace-with-null" }, - { "field-id": 8, "action": "truncate-to-year" }, - { "field-id": 10, "action": "sha-256-global" }, - { "field-id": 12, "action": "mask-alphanum" } - ], - "required-row-filter": { - "type": "eq", - "term": "region", - "value": "US" - } - } + required-row-filter) imposes no restrictions and is equivalent to the + field being absent from the response. + example: + required-column-projections: + - field-id: 4 + action: show-last-4 + - field-id: 6 + action: replace-with-null + - field-id: 8 + action: truncate-to-year + - field-id: 10 + action: sha-256-global + - field-id: 12 + action: mask-alphanum + required-row-filter: + type: eq + term: region + value: US properties: required-column-projections: description: > A list of columns that require specific actions to be applied when reading. + A server must not return an action for a column whose type is not listed in + that action's "Applicable to" set. If this property is present, each listed column must have its specified action applied. Columns not listed in required-column-projections @@ -3742,9 +3735,6 @@ components: the caller. The reader must not silently return raw, partial, or empty results to mask the failure. - 5. Each action defines the output type for its column. For all predefined - actions, the output type matches the input column type. - type: array items: $ref: '#/components/schemas/Action' @@ -3757,14 +3747,7 @@ components: the filter evaluates to FALSE, and no information derived from discarded rows may be included in the query result. - 2. Row filters must be evaluated against the original, untransformed column values. - Required projections must be applied only after row filters are applied. - - 3. If a reader cannot interpret or evaluate a provided filter expression, it must - fail the query with an error to the caller. The reader must not silently return - partial, raw, or empty results to mask the failure. - - 4. If this property is absent, null, or always true then no mandatory filtering is required. + 2. If this property is absent, null, or always true then no mandatory filtering is required. allOf: - $ref: '#/components/schemas/Expression' From 159f47c584d761d10df31d150a73e5029a99a876 Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Sun, 14 Jun 2026 16:01:59 -0700 Subject: [PATCH 29/45] Spec: Rewrite ReadRestrictions property descriptions per rdblue review - required-column-projections: collapse the numbered rule list into declarative prose; consolidate absence semantics into the lead; drop rule 4 (failure semantics now stated once on the parent description); quote 'cc' in the example sentence (L3698, L3700, L3707, L3716). - required-row-filter: collapse to a single declarative paragraph parallel with required-column-projections. The "Iceberg expressions never produce NULL" phrase from L3731 is preserved. --- open-api/rest-catalog-open-api.py | 4 +-- open-api/rest-catalog-open-api.yaml | 45 +++++++++-------------------- 2 files changed, 16 insertions(+), 33 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index 4639f3ed49e2..1db128268abc 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -1697,12 +1697,12 @@ class ReadRestrictions(BaseModel): ) = Field( None, alias='required-column-projections', - description='A list of columns that require specific actions to be applied when reading. A server must not return an action for a column whose type is not listed in that action\'s "Applicable to" set.\nIf this property is present, each listed column must have its specified action applied. Columns not listed in required-column-projections are not subject to any read restrictions.\nWhen this list is present:\n1. For each column listed in required-column-projections, the reader must apply\n the specified action before returning values for that column.\n\n2. The reader must replace all output references to the column with the result\n of the action, presenting the result under the original column name. For\n example, if the action for column cc is mask-alphanum, the reader must\n return the masked value as cc in the query output.\n\n3. A column must appear at most once in required-column-projections.\n4. If a projected column\'s action cannot be evaluated by the reader (including\n unrecognized action types), the reader must fail the query with an error to\n the caller. The reader must not silently return raw, partial, or empty\n results to mask the failure.\n', + description='A list of columns that require specific actions to be applied when reading. A server must not return an action for a column whose type is not listed in that action\'s "Applicable to" set. If absent or empty, no required actions apply.\nEach entry binds an action to a column by field-id. The reader applies the entry\'s action to that column and presents the result under the original column name. For example, if the action for column \'cc\' is mask-alphanum, the reader returns the masked value as \'cc\' in the query output. Columns not listed are read as-is. A field-id must appear at most once.\n', ) required_row_filter: Expression | None = Field( None, alias='required-row-filter', - description='An expression that filters rows in the table that the authenticated principal does not have access to.\n1. The expression must evaluate to a boolean (TRUE or FALSE; Iceberg expressions\n never produce NULL). A reader must discard any row for which\n the filter evaluates to FALSE, and no information derived from discarded rows\n may be included in the query result.\n\n2. If this property is absent, null, or always true then no mandatory filtering is required.\n', + description='An expression that filters out rows the authenticated principal does not have access to. The expression must evaluate to a boolean (TRUE or FALSE; Iceberg expressions never produce NULL). A reader must discard any row for which the filter evaluates to FALSE, and no information derived from discarded rows may be included in the query result. If absent, null, or always true, no mandatory filtering applies.\n', ) diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index 4a279c34d16f..bdc52c5c3a48 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3712,42 +3712,25 @@ components: description: > A list of columns that require specific actions to be applied when reading. A server must not return an action for a column whose type is not listed in - that action's "Applicable to" set. - - If this property is present, each listed column must have its specified - action applied. Columns not listed in required-column-projections - are not subject to any read restrictions. - - When this list is present: - - 1. For each column listed in required-column-projections, the reader must apply - the specified action before returning values for that column. - - 2. The reader must replace all output references to the column with the result - of the action, presenting the result under the original column name. For - example, if the action for column cc is mask-alphanum, the reader must - return the masked value as cc in the query output. - - 3. A column must appear at most once in required-column-projections. - - 4. If a projected column's action cannot be evaluated by the reader (including - unrecognized action types), the reader must fail the query with an error to - the caller. The reader must not silently return raw, partial, or empty - results to mask the failure. - + that action's "Applicable to" set. If absent or empty, no required actions + apply. + + Each entry binds an action to a column by field-id. The reader applies the + entry's action to that column and presents the result under the original + column name. For example, if the action for column 'cc' is mask-alphanum, + the reader returns the masked value as 'cc' in the query output. Columns + not listed are read as-is. A field-id must appear at most once. type: array items: $ref: '#/components/schemas/Action' required-row-filter: description: > - An expression that filters rows in the table that the authenticated principal does not have access to. - - 1. The expression must evaluate to a boolean (TRUE or FALSE; Iceberg expressions - never produce NULL). A reader must discard any row for which - the filter evaluates to FALSE, and no information derived from discarded rows - may be included in the query result. - - 2. If this property is absent, null, or always true then no mandatory filtering is required. + An expression that filters out rows the authenticated principal does not have + access to. The expression must evaluate to a boolean (TRUE or FALSE; Iceberg + expressions never produce NULL). A reader must discard any row for which the + filter evaluates to FALSE, and no information derived from discarded rows may + be included in the query result. If absent, null, or always true, no mandatory + filtering applies. allOf: - $ref: '#/components/schemas/Expression' From cfca72175606c01cb10a3c55f5fb526d09b8902f Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Sun, 14 Jun 2026 16:14:40 -0700 Subject: [PATCH 30/45] Spec: Restore numbered rule lists in ReadRestrictions properties Commit 35c8cba08 flattened the numbered rules in required-column-projections and required-row-filter into prose. That restructuring was not asked for by rdblue; only the wording fixes were. Restore the numbered lists while preserving the wording fixes: - required-column-projections: lead consolidates absence and not-listed semantics; rules 1-3 retained; quote 'cc' in rule 2; rule 4 (failure semantics) deleted because the parent description already covers it for both filters and actions (L3698, L3700, L3707, L3716). - required-row-filter: 2 numbered rules retained; "Iceberg expressions never produce NULL" preserved in rule 1 (L3731). --- open-api/rest-catalog-open-api.py | 4 ++-- open-api/rest-catalog-open-api.yaml | 30 +++++++++++++++++------------ 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index 1db128268abc..8c02dce11379 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -1697,12 +1697,12 @@ class ReadRestrictions(BaseModel): ) = Field( None, alias='required-column-projections', - description='A list of columns that require specific actions to be applied when reading. A server must not return an action for a column whose type is not listed in that action\'s "Applicable to" set. If absent or empty, no required actions apply.\nEach entry binds an action to a column by field-id. The reader applies the entry\'s action to that column and presents the result under the original column name. For example, if the action for column \'cc\' is mask-alphanum, the reader returns the masked value as \'cc\' in the query output. Columns not listed are read as-is. A field-id must appear at most once.\n', + description='A list of columns that require specific actions to be applied when reading. A server must not return an action for a column whose type is not listed in that action\'s "Applicable to" set. If absent or empty, no required actions apply; columns not listed are not subject to any required action.\n1. For each column listed, the reader must apply the specified action before\n returning values for that column.\n\n2. The reader must replace all output references to the column with the result\n of the action, presenting the result under the original column name. For\n example, if the action for column \'cc\' is mask-alphanum, the reader must\n return the masked value as \'cc\' in the query output.\n\n3. A column must appear at most once in required-column-projections.\n', ) required_row_filter: Expression | None = Field( None, alias='required-row-filter', - description='An expression that filters out rows the authenticated principal does not have access to. The expression must evaluate to a boolean (TRUE or FALSE; Iceberg expressions never produce NULL). A reader must discard any row for which the filter evaluates to FALSE, and no information derived from discarded rows may be included in the query result. If absent, null, or always true, no mandatory filtering applies.\n', + description='An expression that filters rows in the table that the authenticated principal does not have access to.\n1. The expression must evaluate to a boolean (TRUE or FALSE; Iceberg expressions\n never produce NULL). A reader must discard any row for which the filter\n evaluates to FALSE, and no information derived from discarded rows may be\n included in the query result.\n\n2. If this property is absent, null, or always true then no mandatory filtering is required.\n', ) diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index bdc52c5c3a48..b1d00e3b8999 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3713,24 +3713,30 @@ components: A list of columns that require specific actions to be applied when reading. A server must not return an action for a column whose type is not listed in that action's "Applicable to" set. If absent or empty, no required actions - apply. + apply; columns not listed are not subject to any required action. - Each entry binds an action to a column by field-id. The reader applies the - entry's action to that column and presents the result under the original - column name. For example, if the action for column 'cc' is mask-alphanum, - the reader returns the masked value as 'cc' in the query output. Columns - not listed are read as-is. A field-id must appear at most once. + 1. For each column listed, the reader must apply the specified action before + returning values for that column. + + 2. The reader must replace all output references to the column with the result + of the action, presenting the result under the original column name. For + example, if the action for column 'cc' is mask-alphanum, the reader must + return the masked value as 'cc' in the query output. + + 3. A column must appear at most once in required-column-projections. type: array items: $ref: '#/components/schemas/Action' required-row-filter: description: > - An expression that filters out rows the authenticated principal does not have - access to. The expression must evaluate to a boolean (TRUE or FALSE; Iceberg - expressions never produce NULL). A reader must discard any row for which the - filter evaluates to FALSE, and no information derived from discarded rows may - be included in the query result. If absent, null, or always true, no mandatory - filtering applies. + An expression that filters rows in the table that the authenticated principal does not have access to. + + 1. The expression must evaluate to a boolean (TRUE or FALSE; Iceberg expressions + never produce NULL). A reader must discard any row for which the filter + evaluates to FALSE, and no information derived from discarded rows may be + included in the query result. + + 2. If this property is absent, null, or always true then no mandatory filtering is required. allOf: - $ref: '#/components/schemas/Expression' From e0770a4c0a0aa492234d4b1a3ae914a467df8f32 Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Sun, 14 Jun 2026 16:49:05 -0700 Subject: [PATCH 31/45] Spec: Per-action wording fixes per rdblue review - MaskAlphanum: rephrase lead so the rules transform code points rather than redacting "code point by code point" (L3772). - MaskToFixedValue: drop redundant "predefined" from lead (L3799); state decimal fixed value as "the unscaled value is 0" (L3809). - ReplaceWithNull: write the negation as "required (non-nullable)" and use "optional" instead of "nullable" in Applicable-to (L3840, L3842). - Replace 13 hard-to-type Unicode arrows with ASCII -> across action examples and NULL-input lines (L3891). - Sha256Global: drop NIST FIPS citation (L3924); simplify int and long output spec by dropping "signed two's complement" (L3935). - Sha256QueryLocal: state that the definition of a query is left to the implementation (L3952). L3825 (nested types) resolved without spec change. L3821 (geometry well-definedness) and L3931 (fixed input support) held for follow-up. --- open-api/rest-catalog-open-api.py | 43 +++++++++++++-------------- open-api/rest-catalog-open-api.yaml | 45 +++++++++++++++-------------- 2 files changed, 45 insertions(+), 43 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index 8c02dce11379..d826d40c68c4 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -597,7 +597,7 @@ class Action(BaseModel): class MaskAlphanum(Action): """ - Redacts the column value Unicode code point by code point using the following rules: + Redacts the column value using the following rules to transform Unicode code points: - Digits (U+0030–U+0039, 0-9) are replaced with 'n' - The following punctuation characters are kept as-is: U+0028 '(' LEFT PARENTHESIS U+0029 ')' RIGHT PARENTHESIS @@ -608,8 +608,8 @@ class MaskAlphanum(Action): - All other Unicode characters (including letters, whitespace, and any punctuation not listed above) are replaced with 'x' - For example: "prashant010696@gmail.com" → "xxxxxxxxnnnnnn@xxxxx.xxx" - NULL input is preserved (NULL → NULL). + For example: "prashant010696@gmail.com" -> "xxxxxxxxnnnnnn@xxxxx.xxx" + NULL input is preserved (NULL -> NULL). Applicable to: string """ @@ -619,8 +619,8 @@ class MaskAlphanum(Action): class MaskToFixedValue(Action): """ - Replaces the column value with a predefined type-specific fixed value. Readers must use exactly the values listed below to ensure consistency across implementations. - Fixed values by type: - boolean: false - int: 0 - long: 0 - float: 0.0 - double: 0.0 - decimal(p, s): 0 (zero with s digits after the decimal point, e.g. 0.00 for decimal(p,2)) - string: "XXXXXXXX" - date: 1970-01-01 - time: 00:00:00 - timestamp: 1970-01-01T00:00:00 - timestamptz: 1970-01-01T00:00:00+00:00 - timestamp_ns: 1970-01-01T00:00:00.000000000 - timestamptz_ns: 1970-01-01T00:00:00.000000000+00:00 - uuid: 00000000-0000-0000-0000-000000000000 - fixed(n): n zero bytes - binary: empty byte sequence - variant: {} - geometry: POINT EMPTY - geography: POINT EMPTY - list: empty list [] - map: empty map {} - struct: struct with each field set to its type-specific default (applied recursively) + Replaces the column value with a type-specific fixed value. Readers must use exactly the values listed below to ensure consistency across implementations. + Fixed values by type: - boolean: false - int: 0 - long: 0 - float: 0.0 - double: 0.0 - decimal(p, s): 0 (the unscaled value is 0) - string: "XXXXXXXX" - date: 1970-01-01 - time: 00:00:00 - timestamp: 1970-01-01T00:00:00 - timestamptz: 1970-01-01T00:00:00+00:00 - timestamp_ns: 1970-01-01T00:00:00.000000000 - timestamptz_ns: 1970-01-01T00:00:00.000000000+00:00 - uuid: 00000000-0000-0000-0000-000000000000 - fixed(n): n zero bytes - binary: empty byte sequence - variant: {} - geometry: POINT EMPTY - geography: POINT EMPTY - list: empty list [] - map: empty map {} - struct: struct with each field set to its type-specific default (applied recursively) NULL input is also replaced with the type-specific fixed value; NULL is not preserved. Applicable to: all data types @@ -631,8 +631,8 @@ class MaskToFixedValue(Action): class ReplaceWithNull(Action): """ - Replaces the entire column value with NULL. NULL input is preserved (NULL → NULL). A server must not return this action for a non-nullable (required) column. - Applicable to: all nullable types + Replaces the entire column value with NULL. NULL input is preserved (NULL -> NULL). A server must not return this action for a required (non-nullable) column. + Applicable to: all optional types """ @@ -642,8 +642,8 @@ class ReplaceWithNull(Action): class ShowFirst4(Action): """ Preserves the first 4 Unicode code points of the column value and redacts the remainder using mask-alphanum rules (see MaskAlphanum for the exact character rules). Values with 4 or fewer Unicode code points are returned unchanged. - For example: "prashant010696@gmail.com" → "prasxxxxnnnnnn@xxxxx.xxx" - NULL input is preserved (NULL → NULL). + For example: "prashant010696@gmail.com" -> "prasxxxxnnnnnn@xxxxx.xxx" + NULL input is preserved (NULL -> NULL). Applicable to: string """ @@ -654,8 +654,8 @@ class ShowFirst4(Action): class ShowLast4(Action): """ Redacts all Unicode code points except the last 4 using mask-alphanum rules (see MaskAlphanum for the exact character rules). Values with 4 or fewer Unicode code points are returned unchanged. - For example: "4111-1111-1111-4444" → "nnnn-nnnn-nnnn-4444" - NULL input is preserved (NULL → NULL). + For example: "4111-1111-1111-4444" -> "nnnn-nnnn-nnnn-4444" + NULL input is preserved (NULL -> NULL). Applicable to: string """ @@ -666,8 +666,8 @@ class ShowLast4(Action): class TruncateToYear(Action): """ Truncates the column value to year precision, setting month, day, and time components to their minimum values. The output type matches the input type. - For example: 2024-07-15 → 2024-01-01 For timestamptz and timestamptz_ns, truncation is performed in UTC. - NULL input is preserved (NULL → NULL). + For example: 2024-07-15 -> 2024-01-01 For timestamptz and timestamptz_ns, truncation is performed in UTC. + NULL input is preserved (NULL -> NULL). Applicable to: date, timestamp, timestamptz, timestamp_ns, timestamptz_ns """ @@ -678,8 +678,8 @@ class TruncateToYear(Action): class TruncateToMonth(Action): """ Truncates the column value to year and month precision, setting day and time components to their minimum values. The output type matches the input type. - For example: 2024-07-15 → 2024-07-01 For timestamptz and timestamptz_ns, truncation is performed in UTC. - NULL input is preserved (NULL → NULL). + For example: 2024-07-15 -> 2024-07-01 For timestamptz and timestamptz_ns, truncation is performed in UTC. + NULL input is preserved (NULL -> NULL). Applicable to: date, timestamp, timestamptz, timestamp_ns, timestamptz_ns """ @@ -689,7 +689,7 @@ class TruncateToMonth(Action): class Sha256Global(Action): """ - Applies SHA-256 as specified in NIST FIPS 180-4. Deterministic across all queries + Applies SHA-256. Deterministic across all queries and engines — the same input always produces the same output. Input-to-bytes encoding by type: @@ -700,11 +700,11 @@ class Sha256Global(Action): Output encoding by type: - string: 64-character lowercase hexadecimal string - - int: first 4 bytes of the digest, read as a signed two's complement little-endian int - - long: first 8 bytes of the digest, read as a signed two's complement little-endian long + - int: first 4 bytes of the digest, read as a little-endian int + - long: first 8 bytes of the digest, read as a little-endian long - binary: the full 32-byte raw SHA-256 digest - NULL input is preserved (NULL → NULL). + NULL input is preserved (NULL -> NULL). Applicable to: string, int, long, binary @@ -716,7 +716,8 @@ class Sha256Global(Action): class Sha256QueryLocal(Action): """ Applies SHA-256 with a per-query random salt, making the output non-deterministic - across queries while remaining consistent within a single query. + across queries while remaining consistent within a single query. The definition + of a query is left to the implementation. The engine must generate a cryptographically random salt of at least 16 bytes for each query. @@ -726,7 +727,7 @@ class Sha256QueryLocal(Action): Output encoding follows the same rules as sha-256-global. - NULL input is preserved (NULL → NULL). + NULL input is preserved (NULL -> NULL). Applicable to: string, int, long, binary diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index b1d00e3b8999..f3b1e382f7d6 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3766,7 +3766,7 @@ components: MaskAlphanum: description: > - Redacts the column value Unicode code point by code point using the following rules: + Redacts the column value using the following rules to transform Unicode code points: - Digits (U+0030–U+0039, 0-9) are replaced with 'n' - The following punctuation characters are kept as-is: @@ -3779,9 +3779,9 @@ components: - All other Unicode characters (including letters, whitespace, and any punctuation not listed above) are replaced with 'x' - For example: "prashant010696@gmail.com" → "xxxxxxxxnnnnnn@xxxxx.xxx" + For example: "prashant010696@gmail.com" -> "xxxxxxxxnnnnnn@xxxxx.xxx" - NULL input is preserved (NULL → NULL). + NULL input is preserved (NULL -> NULL). Applicable to: string allOf: @@ -3793,7 +3793,7 @@ components: MaskToFixedValue: description: > - Replaces the column value with a predefined type-specific fixed value. + Replaces the column value with a type-specific fixed value. Readers must use exactly the values listed below to ensure consistency across implementations. @@ -3803,7 +3803,7 @@ components: - long: 0 - float: 0.0 - double: 0.0 - - decimal(p, s): 0 (zero with s digits after the decimal point, e.g. 0.00 for decimal(p,2)) + - decimal(p, s): 0 (the unscaled value is 0) - string: "XXXXXXXX" - date: 1970-01-01 - time: 00:00:00 @@ -3833,10 +3833,10 @@ components: ReplaceWithNull: description: > - Replaces the entire column value with NULL. NULL input is preserved (NULL → NULL). - A server must not return this action for a non-nullable (required) column. + Replaces the entire column value with NULL. NULL input is preserved (NULL -> NULL). + A server must not return this action for a required (non-nullable) column. - Applicable to: all nullable types + Applicable to: all optional types allOf: - $ref: '#/components/schemas/Action' properties: @@ -3850,9 +3850,9 @@ components: using mask-alphanum rules (see MaskAlphanum for the exact character rules). Values with 4 or fewer Unicode code points are returned unchanged. - For example: "prashant010696@gmail.com" → "prasxxxxnnnnnn@xxxxx.xxx" + For example: "prashant010696@gmail.com" -> "prasxxxxnnnnnn@xxxxx.xxx" - NULL input is preserved (NULL → NULL). + NULL input is preserved (NULL -> NULL). Applicable to: string allOf: @@ -3868,9 +3868,9 @@ components: (see MaskAlphanum for the exact character rules). Values with 4 or fewer Unicode code points are returned unchanged. - For example: "4111-1111-1111-4444" → "nnnn-nnnn-nnnn-4444" + For example: "4111-1111-1111-4444" -> "nnnn-nnnn-nnnn-4444" - NULL input is preserved (NULL → NULL). + NULL input is preserved (NULL -> NULL). Applicable to: string allOf: @@ -3885,10 +3885,10 @@ components: Truncates the column value to year precision, setting month, day, and time components to their minimum values. The output type matches the input type. - For example: 2024-07-15 → 2024-01-01 + For example: 2024-07-15 -> 2024-01-01 For timestamptz and timestamptz_ns, truncation is performed in UTC. - NULL input is preserved (NULL → NULL). + NULL input is preserved (NULL -> NULL). Applicable to: date, timestamp, timestamptz, timestamp_ns, timestamptz_ns allOf: @@ -3903,10 +3903,10 @@ components: Truncates the column value to year and month precision, setting day and time components to their minimum values. The output type matches the input type. - For example: 2024-07-15 → 2024-07-01 + For example: 2024-07-15 -> 2024-07-01 For timestamptz and timestamptz_ns, truncation is performed in UTC. - NULL input is preserved (NULL → NULL). + NULL input is preserved (NULL -> NULL). Applicable to: date, timestamp, timestamptz, timestamp_ns, timestamptz_ns allOf: @@ -3918,7 +3918,7 @@ components: Sha256Global: description: | - Applies SHA-256 as specified in NIST FIPS 180-4. Deterministic across all queries + Applies SHA-256. Deterministic across all queries and engines — the same input always produces the same output. Input-to-bytes encoding by type: @@ -3929,11 +3929,11 @@ components: Output encoding by type: - string: 64-character lowercase hexadecimal string - - int: first 4 bytes of the digest, read as a signed two's complement little-endian int - - long: first 8 bytes of the digest, read as a signed two's complement little-endian long + - int: first 4 bytes of the digest, read as a little-endian int + - long: first 8 bytes of the digest, read as a little-endian long - binary: the full 32-byte raw SHA-256 digest - NULL input is preserved (NULL → NULL). + NULL input is preserved (NULL -> NULL). Applicable to: string, int, long, binary allOf: @@ -3946,7 +3946,8 @@ components: Sha256QueryLocal: description: | Applies SHA-256 with a per-query random salt, making the output non-deterministic - across queries while remaining consistent within a single query. + across queries while remaining consistent within a single query. The definition + of a query is left to the implementation. The engine must generate a cryptographically random salt of at least 16 bytes for each query. @@ -3956,7 +3957,7 @@ components: Output encoding follows the same rules as sha-256-global. - NULL input is preserved (NULL → NULL). + NULL input is preserved (NULL -> NULL). Applicable to: string, int, long, binary allOf: From 670348a646ea15f20caf049e6b5046264b5ae92e Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Mon, 22 Jun 2026 16:56:37 -0700 Subject: [PATCH 32/45] Spec: Replace em-dash with colon in Sha256Global description --- open-api/rest-catalog-open-api.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index f3b1e382f7d6..091dd2b196e5 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3919,7 +3919,7 @@ components: Sha256Global: description: | Applies SHA-256. Deterministic across all queries - and engines — the same input always produces the same output. + and engines: the same input always produces the same output. Input-to-bytes encoding by type: - string: UTF-8 encoded bytes From 2f18458845c4ca5d687a019d955269aed51c6a65 Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Mon, 22 Jun 2026 18:13:58 -0700 Subject: [PATCH 33/45] Spec: Regenerate rest-catalog-open-api.py --- open-api/rest-catalog-open-api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index d826d40c68c4..961cbfc8ac82 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -690,7 +690,7 @@ class TruncateToMonth(Action): class Sha256Global(Action): """ Applies SHA-256. Deterministic across all queries - and engines — the same input always produces the same output. + and engines: the same input always produces the same output. Input-to-bytes encoding by type: - string: UTF-8 encoded bytes From d918afc8f313df9872ce3f586330451205411930 Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Thu, 25 Jun 2026 18:31:46 -0700 Subject: [PATCH 34/45] Spec: Address review nits and restore nested-projection prohibition - Sha256Global / Sha256QueryLocal: replace "engine" with "reader" for consistent normative actor terminology (Steven, Sung) - required-column-projections example: refer to field-id instead of column name to match the field-id-based targeting model (Sung) - ReadRestrictions: restore the explicit prohibition on overlapping projections in the same object, extended to all nested types (struct, list, map). The earlier "outermost takes precedence" rewrite implied inheritance semantics that were not intended and left list/map overlap unaddressed. --- open-api/rest-catalog-open-api.py | 10 +++++----- open-api/rest-catalog-open-api.yaml | 20 +++++++++++--------- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index 961cbfc8ac82..ce64d824252f 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -690,7 +690,7 @@ class TruncateToMonth(Action): class Sha256Global(Action): """ Applies SHA-256. Deterministic across all queries - and engines: the same input always produces the same output. + and readers: the same input always produces the same output. Input-to-bytes encoding by type: - string: UTF-8 encoded bytes @@ -719,9 +719,9 @@ class Sha256QueryLocal(Action): across queries while remaining consistent within a single query. The definition of a query is left to the implementation. - The engine must generate a cryptographically random salt of at least 16 bytes for each query. + The reader must generate a cryptographically random salt of at least 16 bytes for each query. - For each column value, the engine must encode the value to bytes using + For each column value, the reader must encode the value to bytes using sha-256-global's input rules, prepend the per-query salt, and compute SHA-256 over the result. @@ -1677,7 +1677,7 @@ class ReadRestrictions(BaseModel): """ Read restrictions for a table. A reader evaluates the row filter against original, untransformed column values, then applies required-column-projections to the surviving rows. Each action must produce a value of the same type as the input column. If a reader cannot apply any returned restriction (a filter expression or an action), it must fail the query and must not silently return raw, partial, or empty results. - If a projection targets a field, that action governs every value reachable through it; other projections in the same object that target a descendant of that field have no effect. + If a projection targets a nested-typed field (struct, list, or map), other projections in the same ReadRestrictions must not target any descendant field-id (struct subfields, list elements, or map keys/values) at any depth. This avoids ambiguity about which action governs a given leaf value. An empty ReadRestrictions object (no required-column-projections and no required-row-filter) imposes no restrictions and is equivalent to the field being absent from the response. """ @@ -1698,7 +1698,7 @@ class ReadRestrictions(BaseModel): ) = Field( None, alias='required-column-projections', - description='A list of columns that require specific actions to be applied when reading. A server must not return an action for a column whose type is not listed in that action\'s "Applicable to" set. If absent or empty, no required actions apply; columns not listed are not subject to any required action.\n1. For each column listed, the reader must apply the specified action before\n returning values for that column.\n\n2. The reader must replace all output references to the column with the result\n of the action, presenting the result under the original column name. For\n example, if the action for column \'cc\' is mask-alphanum, the reader must\n return the masked value as \'cc\' in the query output.\n\n3. A column must appear at most once in required-column-projections.\n', + description='A list of columns that require specific actions to be applied when reading. A server must not return an action for a column whose type is not listed in that action\'s "Applicable to" set. If absent or empty, no required actions apply; columns not listed are not subject to any required action.\n1. For each column listed, the reader must apply the specified action before\n returning values for that column.\n\n2. The reader must replace all output references to the column with the result\n of the action, presenting the result under the original field-id. For\n example, if the action for field-id `9` is mask-alphanum, the reader must\n return the masked value as field-id `9` in the query output.\n\n3. A column must appear at most once in required-column-projections.\n', ) required_row_filter: Expression | None = Field( None, diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index 091dd2b196e5..bbc78913a538 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3684,9 +3684,11 @@ components: or an action), it must fail the query and must not silently return raw, partial, or empty results. - If a projection targets a field, that action governs every value reachable - through it; other projections in the same object that target a descendant - of that field have no effect. + If a projection targets a nested-typed field (struct, list, or map), + other projections in the same ReadRestrictions must not target any + descendant field-id (struct subfields, list elements, or map keys/values) + at any depth. This avoids ambiguity about which action governs a given + leaf value. An empty ReadRestrictions object (no required-column-projections and no required-row-filter) imposes no restrictions and is equivalent to the @@ -3719,9 +3721,9 @@ components: returning values for that column. 2. The reader must replace all output references to the column with the result - of the action, presenting the result under the original column name. For - example, if the action for column 'cc' is mask-alphanum, the reader must - return the masked value as 'cc' in the query output. + of the action, presenting the result under the original field-id. For + example, if the action for field-id `9` is mask-alphanum, the reader must + return the masked value as field-id `9` in the query output. 3. A column must appear at most once in required-column-projections. type: array @@ -3919,7 +3921,7 @@ components: Sha256Global: description: | Applies SHA-256. Deterministic across all queries - and engines: the same input always produces the same output. + and readers: the same input always produces the same output. Input-to-bytes encoding by type: - string: UTF-8 encoded bytes @@ -3949,9 +3951,9 @@ components: across queries while remaining consistent within a single query. The definition of a query is left to the implementation. - The engine must generate a cryptographically random salt of at least 16 bytes for each query. + The reader must generate a cryptographically random salt of at least 16 bytes for each query. - For each column value, the engine must encode the value to bytes using + For each column value, the reader must encode the value to bytes using sha-256-global's input rules, prepend the per-query salt, and compute SHA-256 over the result. From 8c82b79bcd8faf535af748c3347c1bfc653b58ff Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Mon, 29 Jun 2026 10:46:35 -0700 Subject: [PATCH 35/45] Spec: Forbid projections on map key field-ids Add rule 4 to required-column-projections: a projection must not target a map's key field-id. Applying an action to keys can produce duplicate or null keys, which readers silently coalesce (Parquet/Avro/ORC/Flink all back map values with java.util.Map.put) or reject (map keys are required per spec.md:241), causing data loss. Addresses Sung's review comment on PR #13879. --- open-api/rest-catalog-open-api.py | 2 +- open-api/rest-catalog-open-api.yaml | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index ce64d824252f..86b586a73b83 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -1698,7 +1698,7 @@ class ReadRestrictions(BaseModel): ) = Field( None, alias='required-column-projections', - description='A list of columns that require specific actions to be applied when reading. A server must not return an action for a column whose type is not listed in that action\'s "Applicable to" set. If absent or empty, no required actions apply; columns not listed are not subject to any required action.\n1. For each column listed, the reader must apply the specified action before\n returning values for that column.\n\n2. The reader must replace all output references to the column with the result\n of the action, presenting the result under the original field-id. For\n example, if the action for field-id `9` is mask-alphanum, the reader must\n return the masked value as field-id `9` in the query output.\n\n3. A column must appear at most once in required-column-projections.\n', + description='A list of columns that require specific actions to be applied when reading. A server must not return an action for a column whose type is not listed in that action\'s "Applicable to" set. If absent or empty, no required actions apply; columns not listed are not subject to any required action.\n1. For each column listed, the reader must apply the specified action before\n returning values for that column.\n\n2. The reader must replace all output references to the column with the result\n of the action, presenting the result under the original field-id. For\n example, if the action for field-id `9` is mask-alphanum, the reader must\n return the masked value as field-id `9` in the query output.\n\n3. A column must appear at most once in required-column-projections.\n\n4. A projection must not target a map\'s key field-id. Applying an action\n to keys can produce duplicate or null keys, which readers silently\n coalesce or reject, causing data loss.\n', ) required_row_filter: Expression | None = Field( None, diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index bbc78913a538..c275f86a8878 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3726,6 +3726,10 @@ components: return the masked value as field-id `9` in the query output. 3. A column must appear at most once in required-column-projections. + + 4. A projection must not target a map's key field-id. Applying an action + to keys can produce duplicate or null keys, which readers silently + coalesce or reject, causing data loss. type: array items: $ref: '#/components/schemas/Action' From f5ed94ed45cc0d23b36dd7d20a3ff230079836c1 Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Mon, 29 Jun 2026 10:59:14 -0700 Subject: [PATCH 36/45] Spec: Regenerate rest-catalog-open-api.py with datamodel-code-generator 0.64.1 Pulled in via rebase onto main. The newer generator emits Action discriminator fields as `Literal[X] | None = None` (vs the older 0.60.0 form `Literal[X] = 'x'`) and adds a Summary __pydantic_extra__ rebuild helper. --- open-api/rest-catalog-open-api.py | 91 ++++++++++++++++--------------- 1 file changed, 46 insertions(+), 45 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index 86b586a73b83..d3cb249928fe 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -18,7 +18,7 @@ from __future__ import annotations from datetime import date, timedelta -from typing import Literal +from typing import Dict, Literal from uuid import UUID from pydantic import Base64Str, BaseModel, ConfigDict, Field, RootModel @@ -269,10 +269,13 @@ class Summary(BaseModel): model_config = ConfigDict( extra='allow', ) - __pydantic_extra__: dict[str, str] operation: Literal['append', 'replace', 'overwrite', 'delete'] +Summary.__annotations__['__pydantic_extra__'] = Dict[str, str] +Summary.model_rebuild(force=True) + + class Snapshot(BaseModel): snapshot_id: int = Field(..., alias='snapshot-id') parent_snapshot_id: int | None = Field(None, alias='parent-snapshot-id') @@ -365,17 +368,17 @@ class AssignUUIDUpdate(BaseUpdate): Assigning a UUID to a table/view should only be done when creating the table/view. It is not safe to re-assign the UUID if a table/view already has a UUID assigned """ - action: Literal['assign-uuid'] = 'assign-uuid' + action: Literal['assign-uuid'] uuid: str class UpgradeFormatVersionUpdate(BaseUpdate): - action: Literal['upgrade-format-version'] = 'upgrade-format-version' + action: Literal['upgrade-format-version'] format_version: int = Field(..., alias='format-version') class SetCurrentSchemaUpdate(BaseUpdate): - action: Literal['set-current-schema'] = 'set-current-schema' + action: Literal['set-current-schema'] schema_id: int = Field( ..., alias='schema-id', @@ -384,12 +387,12 @@ class SetCurrentSchemaUpdate(BaseUpdate): class AddPartitionSpecUpdate(BaseUpdate): - action: Literal['add-spec'] = 'add-spec' + action: Literal['add-spec'] spec: PartitionSpec class SetDefaultSpecUpdate(BaseUpdate): - action: Literal['set-default-spec'] = 'set-default-spec' + action: Literal['set-default-spec'] spec_id: int = Field( ..., alias='spec-id', @@ -398,12 +401,12 @@ class SetDefaultSpecUpdate(BaseUpdate): class AddSortOrderUpdate(BaseUpdate): - action: Literal['add-sort-order'] = 'add-sort-order' + action: Literal['add-sort-order'] sort_order: SortOrder = Field(..., alias='sort-order') class SetDefaultSortOrderUpdate(BaseUpdate): - action: Literal['set-default-sort-order'] = 'set-default-sort-order' + action: Literal['set-default-sort-order'] sort_order_id: int = Field( ..., alias='sort-order-id', @@ -412,47 +415,47 @@ class SetDefaultSortOrderUpdate(BaseUpdate): class AddSnapshotUpdate(BaseUpdate): - action: Literal['add-snapshot'] = 'add-snapshot' + action: Literal['add-snapshot'] snapshot: Snapshot class SetSnapshotRefUpdate(BaseUpdate, SnapshotReference): - action: Literal['set-snapshot-ref'] = 'set-snapshot-ref' + action: Literal['set-snapshot-ref'] ref_name: str = Field(..., alias='ref-name') class RemoveSnapshotsUpdate(BaseUpdate): - action: Literal['remove-snapshots'] = 'remove-snapshots' + action: Literal['remove-snapshots'] snapshot_ids: list[int] = Field(..., alias='snapshot-ids') class RemoveSnapshotRefUpdate(BaseUpdate): - action: Literal['remove-snapshot-ref'] = 'remove-snapshot-ref' + action: Literal['remove-snapshot-ref'] ref_name: str = Field(..., alias='ref-name') class SetLocationUpdate(BaseUpdate): - action: Literal['set-location'] = 'set-location' + action: Literal['set-location'] location: str class SetPropertiesUpdate(BaseUpdate): - action: Literal['set-properties'] = 'set-properties' + action: Literal['set-properties'] updates: dict[str, str] class RemovePropertiesUpdate(BaseUpdate): - action: Literal['remove-properties'] = 'remove-properties' + action: Literal['remove-properties'] removals: list[str] class AddViewVersionUpdate(BaseUpdate): - action: Literal['add-view-version'] = 'add-view-version' + action: Literal['add-view-version'] view_version: ViewVersion = Field(..., alias='view-version') class SetCurrentViewVersionUpdate(BaseUpdate): - action: Literal['set-current-view-version'] = 'set-current-view-version' + action: Literal['set-current-view-version'] view_version_id: int = Field( ..., alias='view-version-id', @@ -461,32 +464,32 @@ class SetCurrentViewVersionUpdate(BaseUpdate): class RemoveStatisticsUpdate(BaseUpdate): - action: Literal['remove-statistics'] = 'remove-statistics' + action: Literal['remove-statistics'] snapshot_id: int = Field(..., alias='snapshot-id') class RemovePartitionStatisticsUpdate(BaseUpdate): - action: Literal['remove-partition-statistics'] = 'remove-partition-statistics' + action: Literal['remove-partition-statistics'] snapshot_id: int = Field(..., alias='snapshot-id') class RemovePartitionSpecsUpdate(BaseUpdate): - action: Literal['remove-partition-specs'] = 'remove-partition-specs' + action: Literal['remove-partition-specs'] spec_ids: list[int] = Field(..., alias='spec-ids') class RemoveSchemasUpdate(BaseUpdate): - action: Literal['remove-schemas'] = 'remove-schemas' + action: Literal['remove-schemas'] schema_ids: list[int] = Field(..., alias='schema-ids') class AddEncryptionKeyUpdate(BaseUpdate): - action: Literal['add-encryption-key'] = 'add-encryption-key' + action: Literal['add-encryption-key'] encryption_key: EncryptedKey = Field(..., alias='encryption-key') class RemoveEncryptionKeyUpdate(BaseUpdate): - action: Literal['remove-encryption-key'] = 'remove-encryption-key' + action: Literal['remove-encryption-key'] key_id: str = Field(..., alias='key-id') @@ -519,7 +522,7 @@ class AssertRefSnapshotId(TableRequirement): """ - type: Literal['assert-ref-snapshot-id'] = 'assert-ref-snapshot-id' + type: Literal['assert-ref-snapshot-id'] ref: str snapshot_id: int = Field(..., alias='snapshot-id') @@ -529,7 +532,7 @@ class AssertLastAssignedFieldId(TableRequirement): The table's last assigned column id must match the requirement's `last-assigned-field-id` """ - type: Literal['assert-last-assigned-field-id'] = 'assert-last-assigned-field-id' + type: Literal['assert-last-assigned-field-id'] last_assigned_field_id: int = Field(..., alias='last-assigned-field-id') @@ -538,7 +541,7 @@ class AssertCurrentSchemaId(TableRequirement): The table's current schema id must match the requirement's `current-schema-id` """ - type: Literal['assert-current-schema-id'] = 'assert-current-schema-id' + type: Literal['assert-current-schema-id'] current_schema_id: int = Field(..., alias='current-schema-id') @@ -547,9 +550,7 @@ class AssertLastAssignedPartitionId(TableRequirement): The table's last assigned partition id must match the requirement's `last-assigned-partition-id` """ - type: Literal['assert-last-assigned-partition-id'] = ( - 'assert-last-assigned-partition-id' - ) + type: Literal['assert-last-assigned-partition-id'] last_assigned_partition_id: int = Field(..., alias='last-assigned-partition-id') @@ -558,7 +559,7 @@ class AssertDefaultSpecId(TableRequirement): The table's default spec id must match the requirement's `default-spec-id` """ - type: Literal['assert-default-spec-id'] = 'assert-default-spec-id' + type: Literal['assert-default-spec-id'] default_spec_id: int = Field(..., alias='default-spec-id') @@ -567,7 +568,7 @@ class AssertDefaultSortOrderId(TableRequirement): The table's default sort order id must match the requirement's `default-sort-order-id` """ - type: Literal['assert-default-sort-order-id'] = 'assert-default-sort-order-id' + type: Literal['assert-default-sort-order-id'] default_sort_order_id: int = Field(..., alias='default-sort-order-id') @@ -614,7 +615,7 @@ class MaskAlphanum(Action): """ - action: Literal['mask-alphanum'] = 'mask-alphanum' + action: Literal['mask-alphanum'] | None = None class MaskToFixedValue(Action): @@ -626,7 +627,7 @@ class MaskToFixedValue(Action): """ - action: Literal['mask-to-fixed-value'] = 'mask-to-fixed-value' + action: Literal['mask-to-fixed-value'] | None = None class ReplaceWithNull(Action): @@ -636,7 +637,7 @@ class ReplaceWithNull(Action): """ - action: Literal['replace-with-null'] = 'replace-with-null' + action: Literal['replace-with-null'] | None = None class ShowFirst4(Action): @@ -648,7 +649,7 @@ class ShowFirst4(Action): """ - action: Literal['show-first-4'] = 'show-first-4' + action: Literal['show-first-4'] | None = None class ShowLast4(Action): @@ -660,7 +661,7 @@ class ShowLast4(Action): """ - action: Literal['show-last-4'] = 'show-last-4' + action: Literal['show-last-4'] | None = None class TruncateToYear(Action): @@ -672,7 +673,7 @@ class TruncateToYear(Action): """ - action: Literal['truncate-to-year'] = 'truncate-to-year' + action: Literal['truncate-to-year'] | None = None class TruncateToMonth(Action): @@ -684,7 +685,7 @@ class TruncateToMonth(Action): """ - action: Literal['truncate-to-month'] = 'truncate-to-month' + action: Literal['truncate-to-month'] | None = None class Sha256Global(Action): @@ -710,7 +711,7 @@ class Sha256Global(Action): """ - action: Literal['sha-256-global'] = 'sha-256-global' + action: Literal['sha-256-global'] | None = None class Sha256QueryLocal(Action): @@ -733,7 +734,7 @@ class Sha256QueryLocal(Action): """ - action: Literal['sha-256-query-local'] = 'sha-256-query-local' + action: Literal['sha-256-query-local'] | None = None class LoadCredentialsResponse(BaseModel): @@ -1289,7 +1290,7 @@ class TransformTerm(BaseModel): class SetPartitionStatisticsUpdate(BaseUpdate): - action: Literal['set-partition-statistics'] = 'set-partition-statistics' + action: Literal['set-partition-statistics'] partition_statistics: PartitionStatisticsFile = Field( ..., alias='partition-statistics' ) @@ -1399,7 +1400,7 @@ class Term(RootModel[Reference | TransformTerm]): class SetStatisticsUpdate(BaseUpdate): - action: Literal['set-statistics'] = 'set-statistics' + action: Literal['set-statistics'] snapshot_id: int | None = Field( None, alias='snapshot-id', @@ -1663,7 +1664,7 @@ class ViewMetadata(BaseModel): class AddSchemaUpdate(BaseUpdate): - action: Literal['add-schema'] = 'add-schema' + action: Literal['add-schema'] schema_: Schema = Field(..., alias='schema') last_column_id: int | None = Field( None, @@ -1698,7 +1699,7 @@ class ReadRestrictions(BaseModel): ) = Field( None, alias='required-column-projections', - description='A list of columns that require specific actions to be applied when reading. A server must not return an action for a column whose type is not listed in that action\'s "Applicable to" set. If absent or empty, no required actions apply; columns not listed are not subject to any required action.\n1. For each column listed, the reader must apply the specified action before\n returning values for that column.\n\n2. The reader must replace all output references to the column with the result\n of the action, presenting the result under the original field-id. For\n example, if the action for field-id `9` is mask-alphanum, the reader must\n return the masked value as field-id `9` in the query output.\n\n3. A column must appear at most once in required-column-projections.\n\n4. A projection must not target a map\'s key field-id. Applying an action\n to keys can produce duplicate or null keys, which readers silently\n coalesce or reject, causing data loss.\n', + description='A list of columns that require specific actions to be applied when reading. A server must not return an action for a column whose type is not listed in that action\'s "Applicable to" set. If absent or empty, no required actions apply; columns not listed are not subject to any required action.\n1. For each column listed, the reader must apply the specified action before\n returning values for that column.\n\n2. The reader must replace all output references to the column with the result\n of the action, presenting the result under the original field-id. For\n example, if the action for field-id `9` is mask-alphanum, the reader must\n return the masked value as field-id `9` in the query output.\n\n3. A column must appear at most once in required-column-projections.\n4. A projection must not target a map\'s key field-id. Applying an action\n to keys can produce duplicate or null keys, which readers silently\n coalesce or reject, causing data loss.\n', ) required_row_filter: Expression | None = Field( None, From 422315a1e7d7517a8ccd861ca675b50c95aee902 Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Mon, 29 Jun 2026 11:26:38 -0700 Subject: [PATCH 37/45] Spec: Use 'Iceberg predicates' instead of 'Iceberg expressions' for NULL note Per rdblue review: predicates (not expressions in general) are what cannot produce NULL. Updates the required-row-filter description in both rest-catalog-open-api.yaml and the regenerated .py. --- open-api/rest-catalog-open-api.py | 2 +- open-api/rest-catalog-open-api.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index d3cb249928fe..97bca3dd856c 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -1704,7 +1704,7 @@ class ReadRestrictions(BaseModel): required_row_filter: Expression | None = Field( None, alias='required-row-filter', - description='An expression that filters rows in the table that the authenticated principal does not have access to.\n1. The expression must evaluate to a boolean (TRUE or FALSE; Iceberg expressions\n never produce NULL). A reader must discard any row for which the filter\n evaluates to FALSE, and no information derived from discarded rows may be\n included in the query result.\n\n2. If this property is absent, null, or always true then no mandatory filtering is required.\n', + description='An expression that filters rows in the table that the authenticated principal does not have access to.\n1. The expression must evaluate to a boolean (TRUE or FALSE; Iceberg predicates\n never produce NULL). A reader must discard any row for which the filter\n evaluates to FALSE, and no information derived from discarded rows may be\n included in the query result.\n\n2. If this property is absent, null, or always true then no mandatory filtering is required.\n', ) diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index c275f86a8878..38ee61d0f02a 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3737,7 +3737,7 @@ components: description: > An expression that filters rows in the table that the authenticated principal does not have access to. - 1. The expression must evaluate to a boolean (TRUE or FALSE; Iceberg expressions + 1. The expression must evaluate to a boolean (TRUE or FALSE; Iceberg predicates never produce NULL). A reader must discard any row for which the filter evaluates to FALSE, and no information derived from discarded rows may be included in the query result. From bffa084b91db6866e4a4e0c0635d0800d6321281 Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Mon, 29 Jun 2026 13:04:36 -0700 Subject: [PATCH 38/45] Spec: Exclude geometry/geography from MaskToFixedValue Per rdblue review: POINT EMPTY's WKB encoding is not defined in Iceberg's format spec (Appendix G defers to OGC SFA, which does not specify empty-geometry WKB), and is not formally standardized in GeoParquet, PostGIS, Apache Parquet's geospatial spec, or any other source that Iceberg cites. The "POINT(NaN, NaN)" convention is widely-used implementation lore but never canonicalized. This also aligns with format/spec.md L333 which states that columns of unknown, variant, geometry, and geography types must default to null and explicitly disallows non-null initial-default / write-default values for those types. Drops: - geometry: POINT EMPTY - geography: POINT EMPTY Tightens Applicable-to clause to exclude unknown, geometry, geography. variant is kept in the fixed-value table because Iceberg's Java API defines a canonical empty Variant byte sequence (EMPTY_V1_BUFFER), giving an implementation-anchorable encoding that geometry/geography lack. --- open-api/rest-catalog-open-api.py | 4 ++-- open-api/rest-catalog-open-api.yaml | 4 +--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index 97bca3dd856c..12cf5b78c9a8 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -621,9 +621,9 @@ class MaskAlphanum(Action): class MaskToFixedValue(Action): """ Replaces the column value with a type-specific fixed value. Readers must use exactly the values listed below to ensure consistency across implementations. - Fixed values by type: - boolean: false - int: 0 - long: 0 - float: 0.0 - double: 0.0 - decimal(p, s): 0 (the unscaled value is 0) - string: "XXXXXXXX" - date: 1970-01-01 - time: 00:00:00 - timestamp: 1970-01-01T00:00:00 - timestamptz: 1970-01-01T00:00:00+00:00 - timestamp_ns: 1970-01-01T00:00:00.000000000 - timestamptz_ns: 1970-01-01T00:00:00.000000000+00:00 - uuid: 00000000-0000-0000-0000-000000000000 - fixed(n): n zero bytes - binary: empty byte sequence - variant: {} - geometry: POINT EMPTY - geography: POINT EMPTY - list: empty list [] - map: empty map {} - struct: struct with each field set to its type-specific default (applied recursively) + Fixed values by type: - boolean: false - int: 0 - long: 0 - float: 0.0 - double: 0.0 - decimal(p, s): 0 (the unscaled value is 0) - string: "XXXXXXXX" - date: 1970-01-01 - time: 00:00:00 - timestamp: 1970-01-01T00:00:00 - timestamptz: 1970-01-01T00:00:00+00:00 - timestamp_ns: 1970-01-01T00:00:00.000000000 - timestamptz_ns: 1970-01-01T00:00:00.000000000+00:00 - uuid: 00000000-0000-0000-0000-000000000000 - fixed(n): n zero bytes - binary: empty byte sequence - variant: {} - list: empty list [] - map: empty map {} - struct: struct with each field set to its type-specific default (applied recursively) NULL input is also replaced with the type-specific fixed value; NULL is not preserved. - Applicable to: all data types + Applicable to: all data types except unknown, geometry, and geography. """ diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index 38ee61d0f02a..344ad690ec7c 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3821,15 +3821,13 @@ components: - fixed(n): n zero bytes - binary: empty byte sequence - variant: {} - - geometry: POINT EMPTY - - geography: POINT EMPTY - list: empty list [] - map: empty map {} - struct: struct with each field set to its type-specific default (applied recursively) NULL input is also replaced with the type-specific fixed value; NULL is not preserved. - Applicable to: all data types + Applicable to: all data types except unknown, geometry, and geography. allOf: - $ref: '#/components/schemas/Action' properties: From 98d79a04242df2abc745095c446a809d28b76aac Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Wed, 1 Jul 2026 10:57:38 -0700 Subject: [PATCH 39/45] Spec: Address review comments on read restrictions wording - Use 'reader' consistently in loadTable read-restrictions prose - Rename 'descendant field-id' to 'nested field-id' to match spec terminology - State both actors in the duplicate field-id rule (server must not return, reader must fail) - Forbid mask-to-fixed-value on containers nesting unknown/geometry/geography Regenerate rest-catalog-open-api.py. --- open-api/rest-catalog-open-api.py | 6 +++--- open-api/rest-catalog-open-api.yaml | 19 ++++++++++++------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index 12cf5b78c9a8..e8c5cac7973f 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -623,7 +623,7 @@ class MaskToFixedValue(Action): Replaces the column value with a type-specific fixed value. Readers must use exactly the values listed below to ensure consistency across implementations. Fixed values by type: - boolean: false - int: 0 - long: 0 - float: 0.0 - double: 0.0 - decimal(p, s): 0 (the unscaled value is 0) - string: "XXXXXXXX" - date: 1970-01-01 - time: 00:00:00 - timestamp: 1970-01-01T00:00:00 - timestamptz: 1970-01-01T00:00:00+00:00 - timestamp_ns: 1970-01-01T00:00:00.000000000 - timestamptz_ns: 1970-01-01T00:00:00.000000000+00:00 - uuid: 00000000-0000-0000-0000-000000000000 - fixed(n): n zero bytes - binary: empty byte sequence - variant: {} - list: empty list [] - map: empty map {} - struct: struct with each field set to its type-specific default (applied recursively) NULL input is also replaced with the type-specific fixed value; NULL is not preserved. - Applicable to: all data types except unknown, geometry, and geography. + Applicable to: all data types except unknown, geometry, and geography. Because those three types have no fixed value defined above, a catalog server must not return mask-to-fixed-value for a struct, list, or map that contains a field of type unknown, geometry, or geography at any nesting depth. """ @@ -1678,7 +1678,7 @@ class ReadRestrictions(BaseModel): """ Read restrictions for a table. A reader evaluates the row filter against original, untransformed column values, then applies required-column-projections to the surviving rows. Each action must produce a value of the same type as the input column. If a reader cannot apply any returned restriction (a filter expression or an action), it must fail the query and must not silently return raw, partial, or empty results. - If a projection targets a nested-typed field (struct, list, or map), other projections in the same ReadRestrictions must not target any descendant field-id (struct subfields, list elements, or map keys/values) at any depth. This avoids ambiguity about which action governs a given leaf value. + If a projection targets a nested-typed field (struct, list, or map), other projections in the same ReadRestrictions must not target any nested field-id (struct subfields, list elements, or map keys/values) at any depth. This avoids ambiguity about which action governs a given leaf value. An empty ReadRestrictions object (no required-column-projections and no required-row-filter) imposes no restrictions and is equivalent to the field being absent from the response. """ @@ -1699,7 +1699,7 @@ class ReadRestrictions(BaseModel): ) = Field( None, alias='required-column-projections', - description='A list of columns that require specific actions to be applied when reading. A server must not return an action for a column whose type is not listed in that action\'s "Applicable to" set. If absent or empty, no required actions apply; columns not listed are not subject to any required action.\n1. For each column listed, the reader must apply the specified action before\n returning values for that column.\n\n2. The reader must replace all output references to the column with the result\n of the action, presenting the result under the original field-id. For\n example, if the action for field-id `9` is mask-alphanum, the reader must\n return the masked value as field-id `9` in the query output.\n\n3. A column must appear at most once in required-column-projections.\n4. A projection must not target a map\'s key field-id. Applying an action\n to keys can produce duplicate or null keys, which readers silently\n coalesce or reject, causing data loss.\n', + description='A list of columns that require specific actions to be applied when reading. A server must not return an action for a column whose type is not listed in that action\'s "Applicable to" set. If absent or empty, no required actions apply; columns not listed are not subject to any required action.\n1. For each column listed, the reader must apply the specified action before\n returning values for that column.\n\n2. The reader must replace all output references to the column with the result\n of the action, presenting the result under the original field-id. For\n example, if the action for field-id `9` is mask-alphanum, the reader must\n return the masked value as field-id `9` in the query output.\n\n3. A server must not return more than one projection for the same field-id\n in required-column-projections. If a duplicate field-id appears, the reader\n must fail the query.\n\n4. A projection must not target a map\'s key field-id. Applying an action\n to keys can produce duplicate or null keys, which readers silently\n coalesce or reject, causing data loss.\n', ) required_row_filter: Expression | None = Field( None, diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index 344ad690ec7c..921dd8949929 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -1053,11 +1053,11 @@ paths: key. For example, "urn:ietf:params:oauth:token-type:jwt=". - The response may include a read-restrictions field. A client should support this field; - if it does, it must fail any read against the loaded table when it cannot apply a returned - restriction. This includes unrecognized action or expression types, and actions whose - definition cannot be loaded, parsed, or evaluated. These restrictions apply to every read - performed using this response (including subsequent planTableScan and fetchScanTasks + The response may include a read-restrictions field. A reader that supports read + restrictions must fail any read against the loaded table that cannot apply a returned + restriction in full. This includes unrecognized action or expression types, and actions + whose definition cannot be loaded, parsed, or evaluated. These restrictions apply to every + read performed using this response (including subsequent planTableScan and fetchScanTasks calls), only to the authenticated principal associated with the request, and must not be interpreted as global policy. parameters: @@ -3686,7 +3686,7 @@ components: If a projection targets a nested-typed field (struct, list, or map), other projections in the same ReadRestrictions must not target any - descendant field-id (struct subfields, list elements, or map keys/values) + nested field-id (struct subfields, list elements, or map keys/values) at any depth. This avoids ambiguity about which action governs a given leaf value. @@ -3725,7 +3725,9 @@ components: example, if the action for field-id `9` is mask-alphanum, the reader must return the masked value as field-id `9` in the query output. - 3. A column must appear at most once in required-column-projections. + 3. A server must not return more than one projection for the same field-id + in required-column-projections. If a duplicate field-id appears, the reader + must fail the query. 4. A projection must not target a map's key field-id. Applying an action to keys can produce duplicate or null keys, which readers silently @@ -3828,6 +3830,9 @@ components: NULL input is also replaced with the type-specific fixed value; NULL is not preserved. Applicable to: all data types except unknown, geometry, and geography. + Because those three types have no fixed value defined above, a catalog server + must not return mask-to-fixed-value for a struct, list, or map that contains a + field of type unknown, geometry, or geography at any nesting depth. allOf: - $ref: '#/components/schemas/Action' properties: From d67103b330e72e4bca077cb5fa4155a360600657 Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Wed, 1 Jul 2026 14:16:30 -0700 Subject: [PATCH 40/45] Spec: Address Steven's comments on field-id presence in read schema --- open-api/rest-catalog-open-api.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index 921dd8949929..63474709fcf7 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3732,6 +3732,9 @@ components: 4. A projection must not target a map's key field-id. Applying an action to keys can produce duplicate or null keys, which readers silently coalesce or reject, causing data loss. + + 5. The reader must fail the query if a projection references a field-id + that is not present in the read schema. type: array items: $ref: '#/components/schemas/Action' From 14125c5f3bcf4e321e0b526773d4059c2d8ace2d Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Wed, 1 Jul 2026 14:30:45 -0700 Subject: [PATCH 41/45] Spec: Regenerate rest-catalog-open-api.py --- open-api/rest-catalog-open-api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index e8c5cac7973f..405058ca60f8 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -1699,7 +1699,7 @@ class ReadRestrictions(BaseModel): ) = Field( None, alias='required-column-projections', - description='A list of columns that require specific actions to be applied when reading. A server must not return an action for a column whose type is not listed in that action\'s "Applicable to" set. If absent or empty, no required actions apply; columns not listed are not subject to any required action.\n1. For each column listed, the reader must apply the specified action before\n returning values for that column.\n\n2. The reader must replace all output references to the column with the result\n of the action, presenting the result under the original field-id. For\n example, if the action for field-id `9` is mask-alphanum, the reader must\n return the masked value as field-id `9` in the query output.\n\n3. A server must not return more than one projection for the same field-id\n in required-column-projections. If a duplicate field-id appears, the reader\n must fail the query.\n\n4. A projection must not target a map\'s key field-id. Applying an action\n to keys can produce duplicate or null keys, which readers silently\n coalesce or reject, causing data loss.\n', + description='A list of columns that require specific actions to be applied when reading. A server must not return an action for a column whose type is not listed in that action\'s "Applicable to" set. If absent or empty, no required actions apply; columns not listed are not subject to any required action.\n1. For each column listed, the reader must apply the specified action before\n returning values for that column.\n\n2. The reader must replace all output references to the column with the result\n of the action, presenting the result under the original field-id. For\n example, if the action for field-id `9` is mask-alphanum, the reader must\n return the masked value as field-id `9` in the query output.\n\n3. A server must not return more than one projection for the same field-id\n in required-column-projections. If a duplicate field-id appears, the reader\n must fail the query.\n\n4. A projection must not target a map\'s key field-id. Applying an action\n to keys can produce duplicate or null keys, which readers silently\n coalesce or reject, causing data loss.\n\n5. The reader must fail the query if a projection references a field-id\n that is not present in the read schema.\n', ) required_row_filter: Expression | None = Field( None, From 5af00a6008989a8db6f00fe9b2fbb63e00943ad9 Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Thu, 9 Jul 2026 10:18:41 -0700 Subject: [PATCH 42/45] Spec: Address review comments on ReadRestrictions rule 5 and wording - Reframe rule 5 around what the reader is actually reading per Ryan's sync framing, dropping the invented "read schema" vocabulary that Dan flagged as confusing. - Replace "authenticated principal" and "global policy" (undefined terms per Dan) with "authentication scope" in the loadTable caching note and drop the phrase from required-row-filter. - Scope the fail rule to conforming readers per huaxingao's nit. - Reword the "missing or empty" clause to drop the overloaded "field" per flyrain's nit. - Move the required/non-nullable note into ReplaceWithNull's "Applicable to" line per flyrain's nit. --- open-api/rest-catalog-open-api.yaml | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index 63474709fcf7..4b2cb1bcafcf 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -1058,8 +1058,9 @@ paths: restriction in full. This includes unrecognized action or expression types, and actions whose definition cannot be loaded, parsed, or evaluated. These restrictions apply to every read performed using this response (including subsequent planTableScan and fetchScanTasks - calls), only to the authenticated principal associated with the request, and must not be - interpreted as global policy. + calls), but subsequent requests or requests made from a different authentication scope + may have different restrictions. The response should not be cached outside of the + authenticated scope. parameters: - $ref: '#/components/parameters/data-access' - name: If-None-Match @@ -3680,9 +3681,9 @@ components: A reader evaluates the row filter against original, untransformed column values, then applies required-column-projections to the surviving rows. Each action must produce a value of the same type as the input column. - If a reader cannot apply any returned restriction (a filter expression - or an action), it must fail the query and must not silently return raw, - partial, or empty results. + If a reader that supports read-restrictions cannot apply any returned + restriction (a filter expression or an action), it must fail the query + and must not silently return raw, partial, or empty results. If a projection targets a nested-typed field (struct, list, or map), other projections in the same ReadRestrictions must not target any @@ -3690,9 +3691,8 @@ components: at any depth. This avoids ambiguity about which action governs a given leaf value. - An empty ReadRestrictions object (no required-column-projections and no - required-row-filter) imposes no restrictions and is equivalent to the - field being absent from the response. + A missing or empty ReadRestrictions object (no required-column-projections and no + required-row-filter) imposes no restrictions. example: required-column-projections: - field-id: 4 @@ -3733,14 +3733,14 @@ components: to keys can produce duplicate or null keys, which readers silently coalesce or reject, causing data loss. - 5. The reader must fail the query if a projection references a field-id - that is not present in the read schema. + 5. A reader must enforce projections on the columns it is actually reading. + Projections referencing columns that are not being read do not apply. type: array items: $ref: '#/components/schemas/Action' required-row-filter: description: > - An expression that filters rows in the table that the authenticated principal does not have access to. + An expression that limits which rows the reader may return. 1. The expression must evaluate to a boolean (TRUE or FALSE; Iceberg predicates never produce NULL). A reader must discard any row for which the filter @@ -3845,10 +3845,9 @@ components: ReplaceWithNull: description: > - Replaces the entire column value with NULL. NULL input is preserved (NULL -> NULL). - A server must not return this action for a required (non-nullable) column. + Replaces the column value with NULL. NULL input is preserved (NULL -> NULL). - Applicable to: all optional types + Applicable to: all optional types. Applying to a required (non-nullable) column is invalid. allOf: - $ref: '#/components/schemas/Action' properties: From 92e76c588494d942c8a48e23f8c7c90766b84ae1 Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Thu, 9 Jul 2026 10:18:46 -0700 Subject: [PATCH 43/45] Spec: Regenerate rest-catalog-open-api.py --- open-api/rest-catalog-open-api.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index 405058ca60f8..17194d701cc4 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -632,8 +632,8 @@ class MaskToFixedValue(Action): class ReplaceWithNull(Action): """ - Replaces the entire column value with NULL. NULL input is preserved (NULL -> NULL). A server must not return this action for a required (non-nullable) column. - Applicable to: all optional types + Replaces the column value with NULL. NULL input is preserved (NULL -> NULL). + Applicable to: all optional types. Applying to a required (non-nullable) column is invalid. """ @@ -1677,9 +1677,9 @@ class AddSchemaUpdate(BaseUpdate): class ReadRestrictions(BaseModel): """ Read restrictions for a table. - A reader evaluates the row filter against original, untransformed column values, then applies required-column-projections to the surviving rows. Each action must produce a value of the same type as the input column. If a reader cannot apply any returned restriction (a filter expression or an action), it must fail the query and must not silently return raw, partial, or empty results. + A reader evaluates the row filter against original, untransformed column values, then applies required-column-projections to the surviving rows. Each action must produce a value of the same type as the input column. If a reader that supports read-restrictions cannot apply any returned restriction (a filter expression or an action), it must fail the query and must not silently return raw, partial, or empty results. If a projection targets a nested-typed field (struct, list, or map), other projections in the same ReadRestrictions must not target any nested field-id (struct subfields, list elements, or map keys/values) at any depth. This avoids ambiguity about which action governs a given leaf value. - An empty ReadRestrictions object (no required-column-projections and no required-row-filter) imposes no restrictions and is equivalent to the field being absent from the response. + A missing or empty ReadRestrictions object (no required-column-projections and no required-row-filter) imposes no restrictions. """ @@ -1699,12 +1699,12 @@ class ReadRestrictions(BaseModel): ) = Field( None, alias='required-column-projections', - description='A list of columns that require specific actions to be applied when reading. A server must not return an action for a column whose type is not listed in that action\'s "Applicable to" set. If absent or empty, no required actions apply; columns not listed are not subject to any required action.\n1. For each column listed, the reader must apply the specified action before\n returning values for that column.\n\n2. The reader must replace all output references to the column with the result\n of the action, presenting the result under the original field-id. For\n example, if the action for field-id `9` is mask-alphanum, the reader must\n return the masked value as field-id `9` in the query output.\n\n3. A server must not return more than one projection for the same field-id\n in required-column-projections. If a duplicate field-id appears, the reader\n must fail the query.\n\n4. A projection must not target a map\'s key field-id. Applying an action\n to keys can produce duplicate or null keys, which readers silently\n coalesce or reject, causing data loss.\n\n5. The reader must fail the query if a projection references a field-id\n that is not present in the read schema.\n', + description='A list of columns that require specific actions to be applied when reading. A server must not return an action for a column whose type is not listed in that action\'s "Applicable to" set. If absent or empty, no required actions apply; columns not listed are not subject to any required action.\n1. For each column listed, the reader must apply the specified action before\n returning values for that column.\n\n2. The reader must replace all output references to the column with the result\n of the action, presenting the result under the original field-id. For\n example, if the action for field-id `9` is mask-alphanum, the reader must\n return the masked value as field-id `9` in the query output.\n\n3. A server must not return more than one projection for the same field-id\n in required-column-projections. If a duplicate field-id appears, the reader\n must fail the query.\n\n4. A projection must not target a map\'s key field-id. Applying an action\n to keys can produce duplicate or null keys, which readers silently\n coalesce or reject, causing data loss.\n\n5. A reader must enforce projections on the columns it is actually reading.\n Projections referencing columns that are not being read do not apply.\n', ) required_row_filter: Expression | None = Field( None, alias='required-row-filter', - description='An expression that filters rows in the table that the authenticated principal does not have access to.\n1. The expression must evaluate to a boolean (TRUE or FALSE; Iceberg predicates\n never produce NULL). A reader must discard any row for which the filter\n evaluates to FALSE, and no information derived from discarded rows may be\n included in the query result.\n\n2. If this property is absent, null, or always true then no mandatory filtering is required.\n', + description='An expression that limits which rows the reader may return.\n1. The expression must evaluate to a boolean (TRUE or FALSE; Iceberg predicates\n never produce NULL). A reader must discard any row for which the filter\n evaluates to FALSE, and no information derived from discarded rows may be\n included in the query result.\n\n2. If this property is absent, null, or always true then no mandatory filtering is required.\n', ) From 67aa35a4b716314a91f9cde86bf873122456423b Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Thu, 9 Jul 2026 11:21:22 -0700 Subject: [PATCH 44/45] Spec: Reframe MaskToFixedValue applicable-to as an allowlist MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per laurentgo's review nit on yaml:3835 — flip from an exclusion list ("all types except unknown, geometry, geography") to an inclusion by reference to the "Fixed values by type" list above. New Iceberg types are then implicitly blocked until the spec explicitly extends the fixed-values list to cover them. The container-nesting carve-out is dropped: it's redundant with the MUST NOT clause because a struct containing a field with no fixed value isn't a "type with a fixed value defined above" — the recursive definition simply doesn't apply. --- open-api/rest-catalog-open-api.yaml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index 4b2cb1bcafcf..47df2ed7cdeb 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3832,10 +3832,8 @@ components: NULL input is also replaced with the type-specific fixed value; NULL is not preserved. - Applicable to: all data types except unknown, geometry, and geography. - Because those three types have no fixed value defined above, a catalog server - must not return mask-to-fixed-value for a struct, list, or map that contains a - field of type unknown, geometry, or geography at any nesting depth. + Applicable to: the types with a fixed value defined above. A catalog server + MUST NOT return mask-to-fixed-value for any other type. allOf: - $ref: '#/components/schemas/Action' properties: From 5044d53d4e78c369a86da42fea64495263cf0f32 Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Thu, 9 Jul 2026 11:21:22 -0700 Subject: [PATCH 45/45] Spec: Regenerate rest-catalog-open-api.py --- open-api/rest-catalog-open-api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index 17194d701cc4..730cf0ab875f 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -623,7 +623,7 @@ class MaskToFixedValue(Action): Replaces the column value with a type-specific fixed value. Readers must use exactly the values listed below to ensure consistency across implementations. Fixed values by type: - boolean: false - int: 0 - long: 0 - float: 0.0 - double: 0.0 - decimal(p, s): 0 (the unscaled value is 0) - string: "XXXXXXXX" - date: 1970-01-01 - time: 00:00:00 - timestamp: 1970-01-01T00:00:00 - timestamptz: 1970-01-01T00:00:00+00:00 - timestamp_ns: 1970-01-01T00:00:00.000000000 - timestamptz_ns: 1970-01-01T00:00:00.000000000+00:00 - uuid: 00000000-0000-0000-0000-000000000000 - fixed(n): n zero bytes - binary: empty byte sequence - variant: {} - list: empty list [] - map: empty map {} - struct: struct with each field set to its type-specific default (applied recursively) NULL input is also replaced with the type-specific fixed value; NULL is not preserved. - Applicable to: all data types except unknown, geometry, and geography. Because those three types have no fixed value defined above, a catalog server must not return mask-to-fixed-value for a struct, list, or map that contains a field of type unknown, geometry, or geography at any nesting depth. + Applicable to: the types with a fixed value defined above. A catalog server MUST NOT return mask-to-fixed-value for any other type. """