fix(aws_s3 sink): restrict checksum calculation to fix R2 InvalidRequest

changelog.d/23029_s3_r2_checksum_invalidrequest.fix.md

@@ -0,0 +1,3 @@
+Fixed `InvalidRequest` errors ("You can only specify one non-default checksum at a time") from the `aws_s3` sink and source against S3-compatible services such as Cloudflare R2. The AWS SDK for Rust now calculates an `x-amz-checksum-*` request checksum by default, which conflicts with the `Content-MD5` header Vector already sends and which some S3-compatible providers reject outright. Request checksum calculation and response checksum validation are now restricted to only when required, matching AWS's own guidance for third-party S3-compatible endpoints.
+
+authors: Socialpranker

src/common/s3.rs

@@ -1,4 +1,4 @@
-use aws_sdk_s3::config;
+use aws_sdk_s3::config::{self, RequestChecksumCalculation, ResponseChecksumValidation};
 
 use crate::aws::ClientBuilder;
 
@@ -10,8 +10,54 @@ impl ClientBuilder for S3ClientBuilder {
     type Client = aws_sdk_s3::client::Client;
 
     fn build(&self, config: &aws_types::SdkConfig) -> Self::Client {
-        let builder =
-            config::Builder::from(config).force_path_style(self.force_path_style.unwrap_or(true));
+        let builder = config::Builder::from(config)
+            .force_path_style(self.force_path_style.unwrap_or(true))
+            // The AWS SDK defaults to always calculating a `x-amz-checksum-*` request
+            // checksum and validating response checksums. Vector already computes and
+            // sends its own `Content-MD5` header for `PutObject` requests (see
+            // `s3_common::service::S3Service`), so the SDK's additional checksum is
+            // redundant against AWS S3 and gets rejected outright by S3-compatible
+            // providers such as Cloudflare R2 with
+            // "InvalidRequest: You can only specify one non-default checksum at a time."
+            // Restricting checksum calculation/validation to only when required keeps
+            // AWS S3 behavior correct while fixing compatibility with those providers.
+            // See https://github.com/vectordotdev/vector/issues/23029 and
+            // https://github.com/awslabs/aws-sdk-rust/issues/1240.
+            .request_checksum_calculation(RequestChecksumCalculation::WhenRequired)
+            .response_checksum_validation(ResponseChecksumValidation::WhenRequired);
         aws_sdk_s3::client::Client::from_conf(builder.build())
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use aws_types::{SdkConfig, region::Region};
+
+    use super::*;
+
+    #[test]
+    fn checksum_behavior_defaults_to_when_required() {
+        let sdk_config = SdkConfig::builder()
+            .region(Region::new("us-east-1"))
+            .build();
+
+        let client = S3ClientBuilder {
+            force_path_style: None,
+        }
+        .build(&sdk_config);
+
+        let config = client.config();
+
+        assert_eq!(
+            config.request_checksum_calculation(),
+            Some(&RequestChecksumCalculation::WhenRequired),
+            "request checksum calculation must be restricted to when required so Vector's own \
+             Content-MD5 header isn't paired with an SDK-added x-amz-checksum-* header, which \
+             S3-compatible providers such as Cloudflare R2 reject as an InvalidRequest",
+        );
+        assert_eq!(
+            config.response_checksum_validation(),
+            Some(&ResponseChecksumValidation::WhenRequired),
+        );
+    }
+}