Skip to content

feat(rest): support AWS SigV4 request signing for the REST catalog #2660

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
plusplusjiajia wants to merge 2 commits into
base: main
Choose a base branch
from
Open

feat(rest): support AWS SigV4 request signing for the REST catalog #2660

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8052b4c
feat(rest): support AWS SigV4 request signing for the REST catalog
plusplusjiajia Jun 16, 2026
a4c4d8e
test(rest): add SigV4 signing tests covering the Java suite scenarios
plusplusjiajia Jun 17, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions crates/catalog/rest/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,10 +35,12 @@ chrono = { workspace = true }
http = { workspace = true }
iceberg = { workspace = true }
itertools = { workspace = true }
reqsign-core = "3.0.0"
reqwest = { workspace = true }
serde = { workspace = true }
serde_derive = { workspace = true }
serde_json = { workspace = true }
sha2 = "0.10"
tokio = { workspace = true }
typed-builder = { workspace = true }
uuid = { workspace = true, features = ["v4"] }
Expand Down
38 changes: 38 additions & 0 deletions crates/catalog/rest/public-api.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,22 @@
pub mod iceberg_catalog_rest
pub enum iceberg_catalog_rest::PayloadHashMode
pub iceberg_catalog_rest::PayloadHashMode::IcebergRest
pub iceberg_catalog_rest::PayloadHashMode::StandardAws
impl core::clone::Clone for iceberg_catalog_rest::PayloadHashMode
pub fn iceberg_catalog_rest::PayloadHashMode::clone(&self) -> iceberg_catalog_rest::PayloadHashMode
impl core::cmp::Eq for iceberg_catalog_rest::PayloadHashMode
impl core::cmp::PartialEq for iceberg_catalog_rest::PayloadHashMode
pub fn iceberg_catalog_rest::PayloadHashMode::eq(&self, other: &iceberg_catalog_rest::PayloadHashMode) -> bool
impl core::fmt::Debug for iceberg_catalog_rest::PayloadHashMode
pub fn iceberg_catalog_rest::PayloadHashMode::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result
impl core::marker::Copy for iceberg_catalog_rest::PayloadHashMode
impl core::marker::StructuralPartialEq for iceberg_catalog_rest::PayloadHashMode
pub struct iceberg_catalog_rest::AwsCredentials
pub iceberg_catalog_rest::AwsCredentials::access_key_id: alloc::string::String
pub iceberg_catalog_rest::AwsCredentials::secret_access_key: alloc::string::String
pub iceberg_catalog_rest::AwsCredentials::session_token: core::option::Option<alloc::string::String>
impl core::clone::Clone for iceberg_catalog_rest::AwsCredentials
pub fn iceberg_catalog_rest::AwsCredentials::clone(&self) -> iceberg_catalog_rest::AwsCredentials
pub struct iceberg_catalog_rest::CommitTableRequest
pub iceberg_catalog_rest::CommitTableRequest::identifier: core::option::Option<iceberg::catalog::TableIdent>
pub iceberg_catalog_rest::CommitTableRequest::requirements: alloc::vec::Vec<iceberg::catalog::TableRequirement>
Expand Down Expand Up @@ -231,6 +249,7 @@ pub fn iceberg_catalog_rest::RestCatalog::update_table<'life0, 'async_trait>(&'l
pub struct iceberg_catalog_rest::RestCatalogBuilder
impl iceberg_catalog_rest::RestCatalogBuilder
pub fn iceberg_catalog_rest::RestCatalogBuilder::with_client(self, client: reqwest::async_impl::client::Client) -> Self
pub fn iceberg_catalog_rest::RestCatalogBuilder::with_signer(self, signer: alloc::sync::Arc<dyn iceberg_catalog_rest::HttpRequestSigner>) -> Self
impl core::default::Default for iceberg_catalog_rest::RestCatalogBuilder
pub fn iceberg_catalog_rest::RestCatalogBuilder::default() -> Self
impl core::fmt::Debug for iceberg_catalog_rest::RestCatalogBuilder
Expand All @@ -241,6 +260,15 @@ pub fn iceberg_catalog_rest::RestCatalogBuilder::load(self, name: impl core::con
pub fn iceberg_catalog_rest::RestCatalogBuilder::with_kms_client_factory(self, kms_client_factory: alloc::sync::Arc<dyn iceberg::encryption::kms::factory::KmsClientFactory>) -> Self
pub fn iceberg_catalog_rest::RestCatalogBuilder::with_runtime(self, runtime: iceberg::runtime::Runtime) -> Self
pub fn iceberg_catalog_rest::RestCatalogBuilder::with_storage_factory(self, storage_factory: alloc::sync::Arc<dyn iceberg::io::storage::StorageFactory>) -> Self
pub struct iceberg_catalog_rest::SigV4Signer
impl iceberg_catalog_rest::SigV4Signer
pub fn iceberg_catalog_rest::SigV4Signer::new(credentials: iceberg_catalog_rest::AwsCredentials, region: alloc::string::String, service: alloc::string::String, mode: iceberg_catalog_rest::PayloadHashMode) -> Self
impl core::clone::Clone for iceberg_catalog_rest::SigV4Signer
pub fn iceberg_catalog_rest::SigV4Signer::clone(&self) -> iceberg_catalog_rest::SigV4Signer
impl core::fmt::Debug for iceberg_catalog_rest::SigV4Signer
pub fn iceberg_catalog_rest::SigV4Signer::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result
impl iceberg_catalog_rest::HttpRequestSigner for iceberg_catalog_rest::SigV4Signer
pub fn iceberg_catalog_rest::SigV4Signer::sign(&self, request: &mut reqwest::async_impl::request::Request) -> iceberg::error::Result<()>
pub struct iceberg_catalog_rest::StorageCredential
pub iceberg_catalog_rest::StorageCredential::config: std::collections::hash::map::HashMap<alloc::string::String, alloc::string::String>
pub iceberg_catalog_rest::StorageCredential::prefix: alloc::string::String
Expand Down Expand Up @@ -288,5 +316,15 @@ pub fn iceberg_catalog_rest::UpdateNamespacePropertiesResponse::serialize<__S>(&
impl<'de> serde_core::de::Deserialize<'de> for iceberg_catalog_rest::UpdateNamespacePropertiesResponse
pub fn iceberg_catalog_rest::UpdateNamespacePropertiesResponse::deserialize<__D>(__deserializer: __D) -> core::result::Result<Self, <__D as serde_core::de::Deserializer>::Error> where __D: serde_core::de::Deserializer<'de>
pub const iceberg_catalog_rest::REST_CATALOG_PROP_DISABLE_HEADER_REDACTION: &str
pub const iceberg_catalog_rest::REST_CATALOG_PROP_SIGNING_ACCESS_KEY: &str
pub const iceberg_catalog_rest::REST_CATALOG_PROP_SIGNING_NAME: &str
pub const iceberg_catalog_rest::REST_CATALOG_PROP_SIGNING_REGION: &str
pub const iceberg_catalog_rest::REST_CATALOG_PROP_SIGNING_SECRET_KEY: &str
pub const iceberg_catalog_rest::REST_CATALOG_PROP_SIGNING_SESSION_TOKEN: &str
pub const iceberg_catalog_rest::REST_CATALOG_PROP_SIGV4_ENABLED: &str
pub const iceberg_catalog_rest::REST_CATALOG_PROP_URI: &str
pub const iceberg_catalog_rest::REST_CATALOG_PROP_WAREHOUSE: &str
pub trait iceberg_catalog_rest::HttpRequestSigner: core::marker::Send + core::marker::Sync + core::fmt::Debug
pub fn iceberg_catalog_rest::HttpRequestSigner::sign(&self, request: &mut reqwest::async_impl::request::Request) -> iceberg::error::Result<()>
impl iceberg_catalog_rest::HttpRequestSigner for iceberg_catalog_rest::SigV4Signer
pub fn iceberg_catalog_rest::SigV4Signer::sign(&self, request: &mut reqwest::async_impl::request::Request) -> iceberg::error::Result<()>
250 changes: 250 additions & 0 deletions crates/catalog/rest/src/catalog.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ use typed_builder::TypedBuilder;
use crate::client::{
HttpClient, deserialize_catalog_response, deserialize_unexpected_catalog_error,
};
use crate::signing::{AwsCredentials, HttpRequestSigner, PayloadHashMode, SigV4Signer};
use crate::types::{
CatalogConfig, CommitTableRequest, CommitTableResponse, CreateNamespaceRequest,
CreateTableRequest, ListNamespaceResponse, ListTablesResponse, LoadTableResult,
Expand All @@ -53,6 +54,18 @@ pub const REST_CATALOG_PROP_URI: &str = "uri";
pub const REST_CATALOG_PROP_WAREHOUSE: &str = "warehouse";
/// Disable header redaction in error logs (defaults to false for security)
pub const REST_CATALOG_PROP_DISABLE_HEADER_REDACTION: &str = "disable-header-redaction";
/// Enable AWS SigV4 request signing for the REST catalog.
pub const REST_CATALOG_PROP_SIGV4_ENABLED: &str = "rest.sigv4-enabled";
/// SigV4 signing service name (required when sigv4 is enabled).
pub const REST_CATALOG_PROP_SIGNING_NAME: &str = "rest.signing-name";
/// SigV4 signing region (required when sigv4 is enabled).
pub const REST_CATALOG_PROP_SIGNING_REGION: &str = "rest.signing-region";
/// SigV4 access key id (optional; falls back to the `AWS_ACCESS_KEY_ID` env var).
pub const REST_CATALOG_PROP_SIGNING_ACCESS_KEY: &str = "rest.signing-access-key-id";
/// SigV4 secret access key (optional; falls back to the `AWS_SECRET_ACCESS_KEY` env var).
pub const REST_CATALOG_PROP_SIGNING_SECRET_KEY: &str = "rest.signing-secret-access-key";
/// SigV4 session token (optional; falls back to the `AWS_SESSION_TOKEN` env var).
pub const REST_CATALOG_PROP_SIGNING_SESSION_TOKEN: &str = "rest.signing-session-token";

const ICEBERG_REST_SPEC_VERSION: &str = "0.14.1";
const CARGO_PKG_VERSION: &str = env!("CARGO_PKG_VERSION");
Expand All @@ -76,6 +89,7 @@ impl Default for RestCatalogBuilder {
warehouse: None,
props: HashMap::new(),
client: None,
custom_signer: None,
},
storage_factory: None,
kms_client_factory: None,
Expand Down Expand Up @@ -160,6 +174,12 @@ impl RestCatalogBuilder {
self.config.client = Some(client);
self
}

/// Injects a custom request signer, overriding the `rest.sigv4-*` configuration.
pub fn with_signer(mut self, signer: Arc<dyn HttpRequestSigner>) -> Self {

@CTTY CTTY Jul 7, 2026

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we need a more general design rather than just a signer. some authentication mechanism is token-based.

@plusplusjiajia plusplusjiajia Jul 11, 2026

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@CTTY I dug into how Java structures this, and I think your point is well taken: OAuth2 token handling is hardcoded inside HttpClient today, and this PR adds a second, parallel mechanism that is mutually exclusive with token auth. In Java the two compose — SigV4AuthManager wraps a delegate session, relocates its Authorization header to X-Iceberg-Authorization, then signs — so SigV4-over-OAuth2 is a real combination the current design can't express.

Is something along these lines what you had in mind, mirroring Java's AuthManager/AuthSession?

#[async_trait]
pub trait AuthManager: Debug + Send + Sync {
    /// Session used for catalog-level requests.
    async fn catalog_session(
        &self,
        props: &HashMap<String, String>,
    ) -> Result<Arc<dyn AuthSession>>;
    // room to grow, matching Java: init_session() for the config
    // handshake, table_session() for table-scoped auth, close().
}

#[async_trait]
pub trait AuthSession: Debug + Send + Sync {
    /// Applies authentication to an outgoing request (headers, signing, ...).
    async fn authenticate(&self, request: &mut reqwest::Request) -> Result<()>;
}

with NoopAuthManager / OAuth2Manager (existing logic extracted, behavior unchanged) / SigV4AuthManager (wrapping a delegate, Java-style) as the initial implementations, selected via rest.auth.type or injected through the builder.

If that matches your intuition, my instinct would be to land the interface plus the OAuth2 extraction as a small standalone refactor first, then rework this PR on top as the SigV4 implementation — which would also give #2311 a common landing spot.

@plusplusjiajia plusplusjiajia Jul 12, 2026

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Prototype is up as a draft PR: #2815 — full AuthManager/AuthSession shape with Noop/OAuth2/SigV4 managers; details and known simplifications in the PR description.

self.config.custom_signer = Some(signer);
self
}
}

/// Rest catalog configuration.
Expand All @@ -178,6 +198,9 @@ pub(crate) struct RestCatalogConfig {

#[builder(default)]
client: Option<Client>,

#[builder(default)]
custom_signer: Option<Arc<dyn HttpRequestSigner>>,
}

impl RestCatalogConfig {
Expand Down Expand Up @@ -340,6 +363,100 @@ impl RestCatalogConfig {
.unwrap_or(false)
}

fn sigv4_enabled(&self) -> bool {
self.props
.get(REST_CATALOG_PROP_SIGV4_ENABLED)
.map(|v| v.eq_ignore_ascii_case("true"))
.unwrap_or(false)
}

/// Resolves the request signer: a `with_signer` override wins, otherwise a
/// `SigV4Signer` is built when `rest.sigv4-enabled` is true.
pub(crate) fn resolve_signer(&self) -> Result<Option<Arc<dyn HttpRequestSigner>>> {
if let Some(signer) = &self.custom_signer {
return Ok(Some(signer.clone()));
}
if !self.sigv4_enabled() {
return Ok(None);
}
// SigV4 and OAuth token/credential auth are mutually exclusive.
if self.token().is_some() || self.credential().is_some() {
return Err(Error::new(
ErrorKind::DataInvalid,
format!(
"'{REST_CATALOG_PROP_SIGV4_ENABLED}' cannot be combined with OAuth token/credential authentication"
),
));
}
self.build_sigv4_signer().map(Some)
}

fn build_sigv4_signer(&self) -> Result<Arc<dyn HttpRequestSigner>> {
let region = self
.props
.get(REST_CATALOG_PROP_SIGNING_REGION)
.cloned()
.ok_or_else(|| {
Error::new(
ErrorKind::DataInvalid,
format!(
"'{REST_CATALOG_PROP_SIGNING_REGION}' is required when '{REST_CATALOG_PROP_SIGV4_ENABLED}' is true"
),
)
})?;
let name = self
.props
.get(REST_CATALOG_PROP_SIGNING_NAME)
.cloned()
.ok_or_else(|| {
Error::new(
ErrorKind::DataInvalid,
format!(
"'{REST_CATALOG_PROP_SIGNING_NAME}' is required when '{REST_CATALOG_PROP_SIGV4_ENABLED}' is true"
),
)
})?;
let access_key_id = self
.props
.get(REST_CATALOG_PROP_SIGNING_ACCESS_KEY)
.cloned()
.or_else(|| std::env::var("AWS_ACCESS_KEY_ID").ok())
.ok_or_else(|| {
Error::new(
ErrorKind::DataInvalid,
"missing SigV4 access key id (set 'rest.signing-access-key-id' or AWS_ACCESS_KEY_ID)",
)
})?;
let secret_access_key = self
.props
.get(REST_CATALOG_PROP_SIGNING_SECRET_KEY)
.cloned()
.or_else(|| std::env::var("AWS_SECRET_ACCESS_KEY").ok())
.ok_or_else(|| {
Error::new(
ErrorKind::DataInvalid,
"missing SigV4 secret access key (set 'rest.signing-secret-access-key' or AWS_SECRET_ACCESS_KEY)",
)
})?;
let session_token = self
.props
.get(REST_CATALOG_PROP_SIGNING_SESSION_TOKEN)
.cloned()
.or_else(|| std::env::var("AWS_SESSION_TOKEN").ok());

let credentials = AwsCredentials {
access_key_id,
secret_access_key,
session_token,
};
Ok(Arc::new(SigV4Signer::new(
credentials,
region,
name,
PayloadHashMode::IcebergRest,
)))
}

/// Merge the `RestCatalogConfig` with the a [`CatalogConfig`] (fetched from the REST server).
pub(crate) fn merge_with_config(mut self, mut config: CatalogConfig) -> Self {
if let Some(uri) = config.overrides.remove("uri") {
Expand Down Expand Up @@ -1612,6 +1729,139 @@ mod tests {
list_ns_mock.assert_async().await;
}

#[tokio::test]
async fn test_sigv4_signs_requests() {
let mut server = Server::new_async().await;

let config_mock = create_config_mock(&mut server).await;

// With sigv4 enabled, requests must carry an AWS SigV4 Authorization header.
let list_ns_mock = server
.mock("GET", "/v1/namespaces")
.match_header(
"authorization",
mockito::Matcher::Regex("^AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/".to_string()),
)
.with_body(r#"{"namespaces": []}"#)
.create_async()
.await;

let props = HashMap::from([
(
REST_CATALOG_PROP_SIGV4_ENABLED.to_string(),
"true".to_string(),
),
(
REST_CATALOG_PROP_SIGNING_REGION.to_string(),
"us-east-1".to_string(),
),
(
REST_CATALOG_PROP_SIGNING_NAME.to_string(),
"glue".to_string(),
),
(
REST_CATALOG_PROP_SIGNING_ACCESS_KEY.to_string(),
"AKIDEXAMPLE".to_string(),
),
(
REST_CATALOG_PROP_SIGNING_SECRET_KEY.to_string(),
"wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY".to_string(),
),
]);
let catalog = RestCatalog::new(
RestCatalogConfig::builder()
.uri(server.url())
.props(props)
.build(),
None,
Runtime::current(),
None,
);

let namespaces = catalog.list_namespaces(None).await.unwrap();
assert!(namespaces.is_empty());

config_mock.assert_async().await;
list_ns_mock.assert_async().await;
}

#[test]
fn test_sigv4_rejects_oauth_combination() {
// SigV4 and OAuth token auth are mutually exclusive.
let props = HashMap::from([
(
REST_CATALOG_PROP_SIGV4_ENABLED.to_string(),
"true".to_string(),
),
(
REST_CATALOG_PROP_SIGNING_REGION.to_string(),
"us-east-1".to_string(),
),
(
REST_CATALOG_PROP_SIGNING_NAME.to_string(),
"glue".to_string(),
),
("token".to_string(), "some-oauth-token".to_string()),
]);
let config = RestCatalogConfig::builder()
.uri("http://localhost".to_string())
.props(props)
.build();

let err = config.resolve_signer().unwrap_err();
assert!(err.message().contains("cannot be combined with OAuth"));
}

#[test]
fn test_sigv4_requires_region_and_name() {
// Missing region.
let config = RestCatalogConfig::builder()
.uri("http://localhost".to_string())
.props(HashMap::from([(
REST_CATALOG_PROP_SIGV4_ENABLED.to_string(),
"true".to_string(),
)]))
.build();
let err = config.resolve_signer().unwrap_err();
assert!(err.message().contains(REST_CATALOG_PROP_SIGNING_REGION));

// Region present, name missing.
let config = RestCatalogConfig::builder()
.uri("http://localhost".to_string())
.props(HashMap::from([
(
REST_CATALOG_PROP_SIGV4_ENABLED.to_string(),
"true".to_string(),
),
(
REST_CATALOG_PROP_SIGNING_REGION.to_string(),
"us-east-1".to_string(),
),
]))
.build();
let err = config.resolve_signer().unwrap_err();
assert!(err.message().contains(REST_CATALOG_PROP_SIGNING_NAME));
}

#[test]
fn test_with_signer_overrides_config() {
// A custom signer takes precedence over the sigv4 config.
#[derive(Debug)]
struct StubSigner;
impl HttpRequestSigner for StubSigner {
fn sign(&self, _request: &mut reqwest::Request) -> Result<()> {
Ok(())
}
}

let config = RestCatalogConfig::builder()
.uri("http://localhost".to_string())
.custom_signer(Some(Arc::new(StubSigner)))
.build();

assert!(config.resolve_signer().unwrap().is_some());
}

#[tokio::test]
async fn test_list_namespace_with_pagination() {
let mut server = Server::new_async().await;
Expand Down
Loading
Loading