diff --git a/changelog.d/23029_s3_r2_checksum_invalidrequest.fix.md b/changelog.d/23029_s3_r2_checksum_invalidrequest.fix.md new file mode 100644 index 0000000000000..a7ddc375c20ee --- /dev/null +++ b/changelog.d/23029_s3_r2_checksum_invalidrequest.fix.md @@ -0,0 +1,3 @@ +Fixed `InvalidRequest` errors ("You can only specify one non-default checksum at a time") from the `aws_s3` sink and source against S3-compatible services such as Cloudflare R2. The AWS SDK for Rust now calculates an `x-amz-checksum-*` request checksum by default, which conflicts with the `Content-MD5` header Vector already sends and which some S3-compatible providers reject outright. Request checksum calculation and response checksum validation are now restricted to only when required, matching AWS's own guidance for third-party S3-compatible endpoints. + +authors: Socialpranker diff --git a/src/common/s3.rs b/src/common/s3.rs index 7c7a9810b1850..db5b5bfcc7e5c 100644 --- a/src/common/s3.rs +++ b/src/common/s3.rs @@ -1,4 +1,4 @@ -use aws_sdk_s3::config; +use aws_sdk_s3::config::{self, RequestChecksumCalculation, ResponseChecksumValidation}; use crate::aws::ClientBuilder; @@ -10,8 +10,54 @@ impl ClientBuilder for S3ClientBuilder { type Client = aws_sdk_s3::client::Client; fn build(&self, config: &aws_types::SdkConfig) -> Self::Client { - let builder = - config::Builder::from(config).force_path_style(self.force_path_style.unwrap_or(true)); + let builder = config::Builder::from(config) + .force_path_style(self.force_path_style.unwrap_or(true)) + // The AWS SDK defaults to always calculating a `x-amz-checksum-*` request + // checksum and validating response checksums. Vector already computes and + // sends its own `Content-MD5` header for `PutObject` requests (see + // `s3_common::service::S3Service`), so the SDK's additional checksum is + // redundant against AWS S3 and gets rejected outright by S3-compatible + // providers such as Cloudflare R2 with + // "InvalidRequest: You can only specify one non-default checksum at a time." + // Restricting checksum calculation/validation to only when required keeps + // AWS S3 behavior correct while fixing compatibility with those providers. + // See https://github.com/vectordotdev/vector/issues/23029 and + // https://github.com/awslabs/aws-sdk-rust/issues/1240. + .request_checksum_calculation(RequestChecksumCalculation::WhenRequired) + .response_checksum_validation(ResponseChecksumValidation::WhenRequired); aws_sdk_s3::client::Client::from_conf(builder.build()) } } + +#[cfg(test)] +mod tests { + use aws_types::{SdkConfig, region::Region}; + + use super::*; + + #[test] + fn checksum_behavior_defaults_to_when_required() { + let sdk_config = SdkConfig::builder() + .region(Region::new("us-east-1")) + .build(); + + let client = S3ClientBuilder { + force_path_style: None, + } + .build(&sdk_config); + + let config = client.config(); + + assert_eq!( + config.request_checksum_calculation(), + Some(&RequestChecksumCalculation::WhenRequired), + "request checksum calculation must be restricted to when required so Vector's own \ + Content-MD5 header isn't paired with an SDK-added x-amz-checksum-* header, which \ + S3-compatible providers such as Cloudflare R2 reject as an InvalidRequest", + ); + assert_eq!( + config.response_checksum_validation(), + Some(&ResponseChecksumValidation::WhenRequired), + ); + } +}